2024-09-10T03:10:42.516 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-10T03:10:42.533 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-10T03:10:42.615 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/adking-2024-09-10_02:59:23-orch:cephadm-wip-adk-testing-2024-09-09-1904-distro-default-smithi/7898560 branch: wip-adk-testing-2024-09-09-1904 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: adking@redhat.com first_in_suite: false job_id: '7898560' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: adking-2024-09-10_02:59:23-orch:cephadm-wip-adk-testing-2024-09-09-1904-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-adk-testing-2024-09-09-1904 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: 3a5e3f16720f432aea965c40a8f6eb480095ec6c ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 3a5e3f16720f432aea965c40a8f6eb480095ec6c selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-adk-testing-2024-09-09-1904 sha1: 3a5e3f16720f432aea965c40a8f6eb480095ec6c owner: scheduled_adking@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 6964 sha1: 3a5e3f16720f432aea965c40a8f6eb480095ec6c sleep_before_teardown: 0 subset: 1/20 suite: orch:cephadm suite_branch: wip-adk-testing-2024-09-09-1904 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_3a5e3f16720f432aea965c40a8f6eb480095ec6c/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 3a5e3f16720f432aea965c40a8f6eb480095ec6c targets: smithi130.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDjqsIImA2XDP6eBLXBrah1iOrk4rxYyuX4ol6a/OyV5sTnLUfOIJeBh7I44jZmDFBtq1J+nE+a1jU4Y8BBW2Ng= smithi164.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNG9LtptxUtf4vPLl9erkt1ov8cYjm8OFQa03jDdCngIPv5uPhNiekafMkAHOPxtz4XNP2lL5vd+DySKxDNctIE= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-10_02:59:23 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: adking verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-10T03:10:42.616 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_3a5e3f16720f432aea965c40a8f6eb480095ec6c/qa; will attempt to use it 2024-09-10T03:10:42.616 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_3a5e3f16720f432aea965c40a8f6eb480095ec6c/qa/tasks 2024-09-10T03:10:42.617 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-10T03:10:42.618 INFO:teuthology.task.internal:Checking packages... 2024-09-10T03:10:42.638 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '3a5e3f16720f432aea965c40a8f6eb480095ec6c' 2024-09-10T03:10:42.639 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-10T03:10:42.639 INFO:teuthology.packaging:ref: None 2024-09-10T03:10:42.639 INFO:teuthology.packaging:tag: None 2024-09-10T03:10:42.639 INFO:teuthology.packaging:branch: wip-adk-testing-2024-09-09-1904 2024-09-10T03:10:42.639 INFO:teuthology.packaging:sha1: 3a5e3f16720f432aea965c40a8f6eb480095ec6c 2024-09-10T03:10:42.639 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-adk-testing-2024-09-09-1904 2024-09-10T03:10:42.781 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4885.g3a5e3f16 2024-09-10T03:10:42.782 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-10T03:10:42.790 INFO:teuthology.task.internal:no buildpackages task found 2024-09-10T03:10:42.790 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-10T03:10:42.796 INFO:teuthology.task.internal:Saving configuration 2024-09-10T03:10:42.810 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-10T03:10:42.816 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-10T03:10:42.841 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi130.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-09-10_02:59:23-orch:cephadm-wip-adk-testing-2024-09-09-1904-distro-default-smithi/7898560', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-10 03:05:32.821704', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDjqsIImA2XDP6eBLXBrah1iOrk4rxYyuX4ol6a/OyV5sTnLUfOIJeBh7I44jZmDFBtq1J+nE+a1jU4Y8BBW2Ng='} 2024-09-10T03:10:42.869 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi164.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-09-10_02:59:23-orch:cephadm-wip-adk-testing-2024-09-09-1904-distro-default-smithi/7898560', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-10 03:05:32.823261', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNG9LtptxUtf4vPLl9erkt1ov8cYjm8OFQa03jDdCngIPv5uPhNiekafMkAHOPxtz4XNP2lL5vd+DySKxDNctIE='} 2024-09-10T03:10:42.869 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-10T03:10:42.875 INFO:teuthology.task.internal:roles: ubuntu@smithi130.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-10T03:10:42.875 INFO:teuthology.task.internal:roles: ubuntu@smithi164.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-10T03:10:42.875 INFO:teuthology.run_tasks:Running task console_log... 2024-09-10T03:10:42.933 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f250316fbe0>, signals=[15]) 2024-09-10T03:10:42.933 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-10T03:10:42.939 INFO:teuthology.task.internal:Opening connections... 2024-09-10T03:10:42.939 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi130.front.sepia.ceph.com 2024-09-10T03:10:42.940 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-10T03:10:43.017 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi164.front.sepia.ceph.com 2024-09-10T03:10:43.018 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-10T03:10:43.092 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-10T03:10:43.098 DEBUG:teuthology.orchestra.run.smithi130:> uname -m 2024-09-10T03:10:43.116 INFO:teuthology.orchestra.run.smithi130.stdout:x86_64 2024-09-10T03:10:43.117 DEBUG:teuthology.orchestra.run.smithi130:> cat /etc/os-release 2024-09-10T03:10:43.174 INFO:teuthology.orchestra.run.smithi130.stdout:NAME="CentOS Stream" 2024-09-10T03:10:43.174 INFO:teuthology.orchestra.run.smithi130.stdout:VERSION="9" 2024-09-10T03:10:43.174 INFO:teuthology.orchestra.run.smithi130.stdout:ID="centos" 2024-09-10T03:10:43.174 INFO:teuthology.orchestra.run.smithi130.stdout:ID_LIKE="rhel fedora" 2024-09-10T03:10:43.174 INFO:teuthology.orchestra.run.smithi130.stdout:VERSION_ID="9" 2024-09-10T03:10:43.175 INFO:teuthology.orchestra.run.smithi130.stdout:PLATFORM_ID="platform:el9" 2024-09-10T03:10:43.175 INFO:teuthology.orchestra.run.smithi130.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-10T03:10:43.175 INFO:teuthology.orchestra.run.smithi130.stdout:ANSI_COLOR="0;31" 2024-09-10T03:10:43.175 INFO:teuthology.orchestra.run.smithi130.stdout:LOGO="fedora-logo-icon" 2024-09-10T03:10:43.175 INFO:teuthology.orchestra.run.smithi130.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-10T03:10:43.175 INFO:teuthology.orchestra.run.smithi130.stdout:HOME_URL="https://centos.org/" 2024-09-10T03:10:43.175 INFO:teuthology.orchestra.run.smithi130.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-10T03:10:43.175 INFO:teuthology.orchestra.run.smithi130.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-10T03:10:43.175 INFO:teuthology.orchestra.run.smithi130.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-10T03:10:43.176 INFO:teuthology.lock.ops:Updating smithi130.front.sepia.ceph.com on lock server 2024-09-10T03:10:43.199 DEBUG:teuthology.orchestra.run.smithi164:> uname -m 2024-09-10T03:10:43.217 INFO:teuthology.orchestra.run.smithi164.stdout:x86_64 2024-09-10T03:10:43.217 DEBUG:teuthology.orchestra.run.smithi164:> cat /etc/os-release 2024-09-10T03:10:43.274 INFO:teuthology.orchestra.run.smithi164.stdout:NAME="CentOS Stream" 2024-09-10T03:10:43.274 INFO:teuthology.orchestra.run.smithi164.stdout:VERSION="9" 2024-09-10T03:10:43.274 INFO:teuthology.orchestra.run.smithi164.stdout:ID="centos" 2024-09-10T03:10:43.274 INFO:teuthology.orchestra.run.smithi164.stdout:ID_LIKE="rhel fedora" 2024-09-10T03:10:43.274 INFO:teuthology.orchestra.run.smithi164.stdout:VERSION_ID="9" 2024-09-10T03:10:43.274 INFO:teuthology.orchestra.run.smithi164.stdout:PLATFORM_ID="platform:el9" 2024-09-10T03:10:43.274 INFO:teuthology.orchestra.run.smithi164.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-10T03:10:43.274 INFO:teuthology.orchestra.run.smithi164.stdout:ANSI_COLOR="0;31" 2024-09-10T03:10:43.274 INFO:teuthology.orchestra.run.smithi164.stdout:LOGO="fedora-logo-icon" 2024-09-10T03:10:43.275 INFO:teuthology.orchestra.run.smithi164.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-10T03:10:43.275 INFO:teuthology.orchestra.run.smithi164.stdout:HOME_URL="https://centos.org/" 2024-09-10T03:10:43.275 INFO:teuthology.orchestra.run.smithi164.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-10T03:10:43.275 INFO:teuthology.orchestra.run.smithi164.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-10T03:10:43.275 INFO:teuthology.orchestra.run.smithi164.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-10T03:10:43.275 INFO:teuthology.lock.ops:Updating smithi164.front.sepia.ceph.com on lock server 2024-09-10T03:10:43.299 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-10T03:10:43.308 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-10T03:10:43.314 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-10T03:10:43.314 DEBUG:teuthology.orchestra.run.smithi130:> test '!' -e /home/ubuntu/cephtest 2024-09-10T03:10:43.317 DEBUG:teuthology.orchestra.run.smithi164:> test '!' -e /home/ubuntu/cephtest 2024-09-10T03:10:43.333 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-10T03:10:43.339 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-10T03:10:43.339 DEBUG:teuthology.orchestra.run.smithi130:> test -z $(ls -A /var/lib/ceph) 2024-09-10T03:10:43.374 DEBUG:teuthology.orchestra.run.smithi164:> test -z $(ls -A /var/lib/ceph) 2024-09-10T03:10:43.471 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-10T03:10:43.529 INFO:teuthology.run_tasks:Running task kernel... 2024-09-10T03:10:43.543 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-10T03:10:43.543 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-10T03:10:43.544 DEBUG:teuthology.orchestra.run.smithi130:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-10T03:10:43.544 DEBUG:teuthology.orchestra.run.smithi164:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-10T03:10:43.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-10T03:10:43.562 DEBUG:teuthology.orchestra.run.smithi130:> uname -r 2024-09-10T03:10:43.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-10T03:10:43.563 DEBUG:teuthology.orchestra.run.smithi164:> uname -r 2024-09-10T03:10:43.619 INFO:teuthology.orchestra.run.smithi130.stdout:5.14.0-503.el9.x86_64 2024-09-10T03:10:43.619 INFO:teuthology.task.kernel:Running kernel on smithi130: 5.14.0-503.el9.x86_64 2024-09-10T03:10:43.619 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum install -y kernel 2024-09-10T03:10:43.621 INFO:teuthology.orchestra.run.smithi164.stdout:5.14.0-503.el9.x86_64 2024-09-10T03:10:43.621 INFO:teuthology.task.kernel:Running kernel on smithi164: 5.14.0-503.el9.x86_64 2024-09-10T03:10:43.621 DEBUG:teuthology.orchestra.run.smithi164:> sudo yum install -y kernel 2024-09-10T03:10:47.186 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - BaseOS 4.5 MB/s | 8.2 MB 00:01 2024-09-10T03:10:50.657 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - BaseOS 1.6 MB/s | 8.2 MB 00:05 2024-09-10T03:10:51.387 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - AppStream 6.6 MB/s | 20 MB 00:03 2024-09-10T03:10:55.624 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - AppStream 5.3 MB/s | 20 MB 00:03 2024-09-10T03:10:57.978 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - CRB 2.3 MB/s | 6.5 MB 00:02 2024-09-10T03:10:59.471 INFO:teuthology.orchestra.run.smithi164.stdout:CentOS Stream 9 - Extras packages 63 kB/s | 19 kB 00:00 2024-09-10T03:11:00.291 INFO:teuthology.orchestra.run.smithi164.stdout:Extra Packages for Enterprise Linux 32 MB/s | 23 MB 00:00 2024-09-10T03:11:01.141 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - CRB 3.8 MB/s | 6.5 MB 00:01 2024-09-10T03:11:02.608 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - Extras packages 69 kB/s | 19 kB 00:00 2024-09-10T03:11:03.429 INFO:teuthology.orchestra.run.smithi130.stdout:Extra Packages for Enterprise Linux 32 MB/s | 23 MB 00:00 2024-09-10T03:11:05.681 INFO:teuthology.orchestra.run.smithi164.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-09-10T03:11:07.703 INFO:teuthology.orchestra.run.smithi164.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-10T03:11:07.704 INFO:teuthology.orchestra.run.smithi164.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-10T03:11:07.800 INFO:teuthology.orchestra.run.smithi164.stdout:Dependencies resolved. 2024-09-10T03:11:07.807 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-10T03:11:07.808 INFO:teuthology.orchestra.run.smithi164.stdout: Package Architecture Version Repository Size 2024-09-10T03:11:07.808 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-10T03:11:07.808 INFO:teuthology.orchestra.run.smithi164.stdout:Installing: 2024-09-10T03:11:07.808 INFO:teuthology.orchestra.run.smithi164.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-10T03:11:07.808 INFO:teuthology.orchestra.run.smithi164.stdout:Installing dependencies: 2024-09-10T03:11:07.808 INFO:teuthology.orchestra.run.smithi164.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-10T03:11:07.808 INFO:teuthology.orchestra.run.smithi164.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-10T03:11:07.809 INFO:teuthology.orchestra.run.smithi164.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-10T03:11:07.809 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:11:07.809 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction Summary 2024-09-10T03:11:07.809 INFO:teuthology.orchestra.run.smithi164.stdout:================================================================================ 2024-09-10T03:11:07.809 INFO:teuthology.orchestra.run.smithi164.stdout:Install 4 Packages 2024-09-10T03:11:07.809 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:11:07.810 INFO:teuthology.orchestra.run.smithi164.stdout:Total download size: 79 M 2024-09-10T03:11:07.810 INFO:teuthology.orchestra.run.smithi164.stdout:Installed size: 126 M 2024-09-10T03:11:07.810 INFO:teuthology.orchestra.run.smithi164.stdout:Downloading Packages: 2024-09-10T03:11:08.465 INFO:teuthology.orchestra.run.smithi164.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 42 kB/s | 23 kB 00:00 2024-09-10T03:11:08.860 INFO:teuthology.orchestra.run.smithi130.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-09-10T03:11:09.315 INFO:teuthology.orchestra.run.smithi164.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 11 MB/s | 16 MB 00:01 2024-09-10T03:11:09.991 INFO:teuthology.orchestra.run.smithi164.stdout:(3/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 17 MB/s | 35 MB 00:02 2024-09-10T03:11:10.657 INFO:teuthology.orchestra.run.smithi164.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 13 MB/s | 29 MB 00:02 2024-09-10T03:11:10.658 INFO:teuthology.orchestra.run.smithi164.stdout:-------------------------------------------------------------------------------- 2024-09-10T03:11:10.658 INFO:teuthology.orchestra.run.smithi164.stdout:Total 28 MB/s | 79 MB 00:02 2024-09-10T03:11:10.883 INFO:teuthology.orchestra.run.smithi130.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-10T03:11:10.883 INFO:teuthology.orchestra.run.smithi130.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-10T03:11:10.979 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-09-10T03:11:10.994 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-09-10T03:11:10.994 INFO:teuthology.orchestra.run.smithi130.stdout: Package Architecture Version Repository Size 2024-09-10T03:11:10.995 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-09-10T03:11:10.995 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2024-09-10T03:11:10.995 INFO:teuthology.orchestra.run.smithi130.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-10T03:11:10.995 INFO:teuthology.orchestra.run.smithi130.stdout:Installing dependencies: 2024-09-10T03:11:10.995 INFO:teuthology.orchestra.run.smithi130.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-10T03:11:10.995 INFO:teuthology.orchestra.run.smithi130.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-10T03:11:10.995 INFO:teuthology.orchestra.run.smithi130.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-10T03:11:10.995 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:11:10.995 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-09-10T03:11:10.995 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-09-10T03:11:10.996 INFO:teuthology.orchestra.run.smithi130.stdout:Install 4 Packages 2024-09-10T03:11:10.996 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:11:10.996 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 79 M 2024-09-10T03:11:10.997 INFO:teuthology.orchestra.run.smithi130.stdout:Installed size: 126 M 2024-09-10T03:11:10.997 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-09-10T03:11:11.120 INFO:teuthology.orchestra.run.smithi130.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 310 kB/s | 23 kB 00:00 2024-09-10T03:11:11.341 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction check 2024-09-10T03:11:11.699 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction check succeeded. 2024-09-10T03:11:11.699 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction test 2024-09-10T03:11:12.453 INFO:teuthology.orchestra.run.smithi130.stdout:(2/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 25 MB/s | 35 MB 00:01 2024-09-10T03:11:12.671 INFO:teuthology.orchestra.run.smithi164.stdout:Transaction test succeeded. 2024-09-10T03:11:12.672 INFO:teuthology.orchestra.run.smithi164.stdout:Running transaction 2024-09-10T03:11:14.087 INFO:teuthology.orchestra.run.smithi130.stdout:(3/4): kernel-modules-core-5.14.0-505.el9.x86_6 9.6 MB/s | 29 MB 00:02 2024-09-10T03:11:14.904 INFO:teuthology.orchestra.run.smithi130.stdout:(4/4): kernel-core-5.14.0-505.el9.x86_64.rpm 4.1 MB/s | 16 MB 00:03 2024-09-10T03:11:14.904 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-09-10T03:11:14.904 INFO:teuthology.orchestra.run.smithi130.stdout:Total 20 MB/s | 79 MB 00:03 2024-09-10T03:11:15.128 INFO:teuthology.orchestra.run.smithi164.stdout: Preparing : 1/1 2024-09-10T03:11:15.587 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-09-10T03:11:15.775 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-10T03:11:15.874 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-10T03:11:15.940 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-09-10T03:11:15.941 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-09-10T03:11:16.869 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-09-10T03:11:16.869 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-09-10T03:11:18.688 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-10T03:11:18.819 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-10T03:11:19.315 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-09-10T03:11:20.221 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-10T03:11:20.305 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-10T03:11:23.078 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-10T03:11:23.217 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-10T03:11:23.640 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-10T03:11:23.744 INFO:teuthology.orchestra.run.smithi164.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-10T03:11:28.063 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-10T03:11:28.184 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-10T03:11:28.597 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-10T03:11:33.043 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-10T03:12:06.821 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-10T03:12:06.867 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-10T03:12:07.196 INFO:teuthology.orchestra.run.smithi164.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-10T03:12:07.197 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-10T03:12:07.197 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-10T03:12:07.197 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-10T03:12:08.064 INFO:teuthology.orchestra.run.smithi164.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-10T03:12:08.064 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:12:08.064 INFO:teuthology.orchestra.run.smithi164.stdout:Installed: 2024-09-10T03:12:08.064 INFO:teuthology.orchestra.run.smithi164.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-10T03:12:08.064 INFO:teuthology.orchestra.run.smithi164.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-10T03:12:08.064 INFO:teuthology.orchestra.run.smithi164.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-10T03:12:08.064 INFO:teuthology.orchestra.run.smithi164.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-10T03:12:08.064 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:12:08.064 INFO:teuthology.orchestra.run.smithi164.stdout:Complete! 2024-09-10T03:12:08.416 DEBUG:teuthology.orchestra.run.smithi164:> rpm -q kernel | sort -rV | head -n 1 2024-09-10T03:12:08.498 INFO:teuthology.orchestra.run.smithi164.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-10T03:12:08.498 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-10T03:12:08.498 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-10T03:12:08.498 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-10T03:12:08.498 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-10T03:12:08.498 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-10T03:12:08.499 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi164.front.sepia.ceph.com, path=None, version=distro) 2024-09-10T03:12:08.499 DEBUG:teuthology.orchestra.run.smithi164:> rpm -q kernel | sort -rV | head -n 1 2024-09-10T03:12:08.557 INFO:teuthology.orchestra.run.smithi164.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-10T03:12:08.557 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-10T03:12:08.558 DEBUG:teuthology.orchestra.run.smithi164:> sudo rpm -qi grub2-tools 2024-09-10T03:12:08.589 INFO:teuthology.orchestra.run.smithi164.stdout:Name : grub2-tools 2024-09-10T03:12:08.589 INFO:teuthology.orchestra.run.smithi164.stdout:Epoch : 1 2024-09-10T03:12:08.589 INFO:teuthology.orchestra.run.smithi164.stdout:Version : 2.06 2024-09-10T03:12:08.589 INFO:teuthology.orchestra.run.smithi164.stdout:Release : 82.el9 2024-09-10T03:12:08.589 INFO:teuthology.orchestra.run.smithi164.stdout:Architecture: x86_64 2024-09-10T03:12:08.589 INFO:teuthology.orchestra.run.smithi164.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-10T03:12:08.589 INFO:teuthology.orchestra.run.smithi164.stdout:Group : Unspecified 2024-09-10T03:12:08.589 INFO:teuthology.orchestra.run.smithi164.stdout:Size : 8274670 2024-09-10T03:12:08.590 INFO:teuthology.orchestra.run.smithi164.stdout:License : GPLv3+ 2024-09-10T03:12:08.590 INFO:teuthology.orchestra.run.smithi164.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-10T03:12:08.590 INFO:teuthology.orchestra.run.smithi164.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-10T03:12:08.590 INFO:teuthology.orchestra.run.smithi164.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-10T03:12:08.590 INFO:teuthology.orchestra.run.smithi164.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-10T03:12:08.590 INFO:teuthology.orchestra.run.smithi164.stdout:Packager : builder@centos.org 2024-09-10T03:12:08.590 INFO:teuthology.orchestra.run.smithi164.stdout:Vendor : CentOS 2024-09-10T03:12:08.590 INFO:teuthology.orchestra.run.smithi164.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-10T03:12:08.590 INFO:teuthology.orchestra.run.smithi164.stdout:Summary : Support tools for GRUB. 2024-09-10T03:12:08.590 INFO:teuthology.orchestra.run.smithi164.stdout:Description : 2024-09-10T03:12:08.590 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:12:08.590 INFO:teuthology.orchestra.run.smithi164.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-10T03:12:08.590 INFO:teuthology.orchestra.run.smithi164.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-10T03:12:08.590 INFO:teuthology.orchestra.run.smithi164.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-10T03:12:08.591 INFO:teuthology.orchestra.run.smithi164.stdout:hardware devices. 2024-09-10T03:12:08.591 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:12:08.591 INFO:teuthology.orchestra.run.smithi164.stdout:This subpackage provides tools for support of all platforms. 2024-09-10T03:12:08.591 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-10T03:12:08.592 INFO:teuthology.task.kernel:Updating grub on smithi164 to boot 5.14.0-505.el9.x86_64 2024-09-10T03:12:08.592 DEBUG:teuthology.orchestra.run.smithi164:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-10T03:12:09.260 INFO:teuthology.orchestra.run.smithi164.stderr:Generating grub configuration file ... 2024-09-10T03:12:10.552 INFO:teuthology.orchestra.run.smithi164.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-10T03:12:10.587 INFO:teuthology.orchestra.run.smithi164.stderr:done 2024-09-10T03:12:10.589 DEBUG:teuthology.orchestra.run.smithi164:> mktemp 2024-09-10T03:12:10.605 INFO:teuthology.orchestra.run.smithi164.stdout:/tmp/tmp.A2rLFOTHj8 2024-09-10T03:12:10.606 DEBUG:teuthology.orchestra.run.smithi164:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.A2rLFOTHj8 2024-09-10T03:12:10.673 DEBUG:teuthology.orchestra.run.smithi164:> sudo chmod 0666 /tmp/tmp.A2rLFOTHj8 2024-09-10T03:12:10.811 DEBUG:teuthology.orchestra.remote:smithi164:/tmp/tmp.A2rLFOTHj8 is 6KB 2024-09-10T03:12:10.860 DEBUG:teuthology.orchestra.run.smithi164:> rm -fr /tmp/tmp.A2rLFOTHj8 2024-09-10T03:12:10.874 DEBUG:teuthology.orchestra.run.smithi164:> sudo /bin/ls /boot/loader/entries || true 2024-09-10T03:12:10.946 INFO:teuthology.orchestra.run.smithi164.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-10T03:12:10.946 INFO:teuthology.orchestra.run.smithi164.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-10T03:12:10.946 INFO:teuthology.orchestra.run.smithi164.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-10T03:12:10.946 INFO:teuthology.orchestra.run.smithi164.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-10T03:12:10.947 DEBUG:teuthology.orchestra.run.smithi164:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-10T03:12:11.119 DEBUG:teuthology.orchestra.run.smithi164:> sudo shutdown -r now 2024-09-10T03:12:11.233 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-10T03:12:11.273 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-10T03:12:11.583 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-10T03:12:11.583 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-10T03:12:11.583 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-10T03:12:11.583 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-10T03:12:12.468 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-10T03:12:12.469 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:12:12.469 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2024-09-10T03:12:12.469 INFO:teuthology.orchestra.run.smithi130.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-10T03:12:12.469 INFO:teuthology.orchestra.run.smithi130.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-10T03:12:12.469 INFO:teuthology.orchestra.run.smithi130.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-10T03:12:12.469 INFO:teuthology.orchestra.run.smithi130.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-10T03:12:12.469 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:12:12.469 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-09-10T03:12:12.828 DEBUG:teuthology.orchestra.run.smithi130:> rpm -q kernel | sort -rV | head -n 1 2024-09-10T03:12:12.917 INFO:teuthology.orchestra.run.smithi130.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-10T03:12:12.917 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-10T03:12:12.917 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-10T03:12:12.917 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-10T03:12:12.917 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-10T03:12:12.918 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-10T03:12:12.918 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi130.front.sepia.ceph.com, path=None, version=distro) 2024-09-10T03:12:12.918 DEBUG:teuthology.orchestra.run.smithi130:> rpm -q kernel | sort -rV | head -n 1 2024-09-10T03:12:12.976 INFO:teuthology.orchestra.run.smithi130.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-10T03:12:12.977 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-10T03:12:12.977 DEBUG:teuthology.orchestra.run.smithi130:> sudo rpm -qi grub2-tools 2024-09-10T03:12:13.009 INFO:teuthology.orchestra.run.smithi130.stdout:Name : grub2-tools 2024-09-10T03:12:13.009 INFO:teuthology.orchestra.run.smithi130.stdout:Epoch : 1 2024-09-10T03:12:13.009 INFO:teuthology.orchestra.run.smithi130.stdout:Version : 2.06 2024-09-10T03:12:13.009 INFO:teuthology.orchestra.run.smithi130.stdout:Release : 82.el9 2024-09-10T03:12:13.009 INFO:teuthology.orchestra.run.smithi130.stdout:Architecture: x86_64 2024-09-10T03:12:13.009 INFO:teuthology.orchestra.run.smithi130.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-10T03:12:13.009 INFO:teuthology.orchestra.run.smithi130.stdout:Group : Unspecified 2024-09-10T03:12:13.009 INFO:teuthology.orchestra.run.smithi130.stdout:Size : 8274670 2024-09-10T03:12:13.009 INFO:teuthology.orchestra.run.smithi130.stdout:License : GPLv3+ 2024-09-10T03:12:13.010 INFO:teuthology.orchestra.run.smithi130.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-10T03:12:13.010 INFO:teuthology.orchestra.run.smithi130.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-10T03:12:13.010 INFO:teuthology.orchestra.run.smithi130.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-10T03:12:13.010 INFO:teuthology.orchestra.run.smithi130.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-10T03:12:13.010 INFO:teuthology.orchestra.run.smithi130.stdout:Packager : builder@centos.org 2024-09-10T03:12:13.010 INFO:teuthology.orchestra.run.smithi130.stdout:Vendor : CentOS 2024-09-10T03:12:13.010 INFO:teuthology.orchestra.run.smithi130.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-10T03:12:13.010 INFO:teuthology.orchestra.run.smithi130.stdout:Summary : Support tools for GRUB. 2024-09-10T03:12:13.010 INFO:teuthology.orchestra.run.smithi130.stdout:Description : 2024-09-10T03:12:13.010 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:12:13.010 INFO:teuthology.orchestra.run.smithi130.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-10T03:12:13.010 INFO:teuthology.orchestra.run.smithi130.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-10T03:12:13.010 INFO:teuthology.orchestra.run.smithi130.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-10T03:12:13.010 INFO:teuthology.orchestra.run.smithi130.stdout:hardware devices. 2024-09-10T03:12:13.011 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:12:13.011 INFO:teuthology.orchestra.run.smithi130.stdout:This subpackage provides tools for support of all platforms. 2024-09-10T03:12:13.011 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-10T03:12:13.012 INFO:teuthology.task.kernel:Updating grub on smithi130 to boot 5.14.0-505.el9.x86_64 2024-09-10T03:12:13.012 DEBUG:teuthology.orchestra.run.smithi130:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-10T03:12:13.696 INFO:teuthology.orchestra.run.smithi130.stderr:Generating grub configuration file ... 2024-09-10T03:12:15.000 INFO:teuthology.orchestra.run.smithi130.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-10T03:12:15.041 INFO:teuthology.orchestra.run.smithi130.stderr:done 2024-09-10T03:12:15.043 DEBUG:teuthology.orchestra.run.smithi130:> mktemp 2024-09-10T03:12:15.073 INFO:teuthology.orchestra.run.smithi130.stdout:/tmp/tmp.svwqFPCDZk 2024-09-10T03:12:15.073 DEBUG:teuthology.orchestra.run.smithi130:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.svwqFPCDZk 2024-09-10T03:12:15.139 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod 0666 /tmp/tmp.svwqFPCDZk 2024-09-10T03:12:15.280 DEBUG:teuthology.orchestra.remote:smithi130:/tmp/tmp.svwqFPCDZk is 6KB 2024-09-10T03:12:15.329 DEBUG:teuthology.orchestra.run.smithi130:> rm -fr /tmp/tmp.svwqFPCDZk 2024-09-10T03:12:15.344 DEBUG:teuthology.orchestra.run.smithi130:> sudo /bin/ls /boot/loader/entries || true 2024-09-10T03:12:15.407 INFO:teuthology.orchestra.run.smithi130.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-10T03:12:15.407 INFO:teuthology.orchestra.run.smithi130.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-10T03:12:15.407 INFO:teuthology.orchestra.run.smithi130.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-10T03:12:15.407 INFO:teuthology.orchestra.run.smithi130.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-10T03:12:15.409 DEBUG:teuthology.orchestra.run.smithi130:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-10T03:12:15.559 DEBUG:teuthology.orchestra.run.smithi130:> sudo shutdown -r now 2024-09-10T03:12:41.141 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-10T03:12:41.142 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi164.front.sepia.ceph.com' 2024-09-10T03:12:41.143 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-10T03:12:45.566 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-10T03:12:45.566 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi130.front.sepia.ceph.com' 2024-09-10T03:12:45.567 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-10T03:12:59.713 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.164 2024-09-10T03:13:04.061 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.130 2024-09-10T03:13:08.717 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi164.front.sepia.ceph.com' 2024-09-10T03:13:08.717 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-10T03:13:11.777 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.164 2024-09-10T03:13:13.061 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi130.front.sepia.ceph.com' 2024-09-10T03:13:13.062 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-10T03:13:16.125 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.130 2024-09-10T03:13:23.785 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi164.front.sepia.ceph.com' 2024-09-10T03:13:23.786 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-10T03:13:28.130 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi130.front.sepia.ceph.com' 2024-09-10T03:13:28.130 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-10T03:14:00.133 DEBUG:teuthology.orchestra.run.smithi130:> true 2024-09-10T03:14:00.545 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi130.front.sepia.ceph.com' 2024-09-10T03:14:00.545 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-505.el9.x86_64"... 2024-09-10T03:14:00.546 DEBUG:teuthology.orchestra.run.smithi130:> uname -r 2024-09-10T03:14:00.560 INFO:teuthology.orchestra.run.smithi130.stdout:5.14.0-505.el9.x86_64 2024-09-10T03:14:00.560 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-10T03:14:00.560 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-10T03:14:00.560 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-10T03:14:01.561 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-10T03:14:01.561 DEBUG:teuthology.orchestra.run.smithi130:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-10T03:14:01.664 INFO:teuthology.orchestra.run.smithi130.stdout:ttyS1 2024-09-10T03:14:01.699 DEBUG:teuthology.parallel:result is None 2024-09-10T03:14:23.788 DEBUG:teuthology.orchestra.remote:timed out 2024-09-10T03:14:38.790 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi164.front.sepia.ceph.com' 2024-09-10T03:14:38.791 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-10T03:14:39.110 DEBUG:teuthology.orchestra.run.smithi164:> true 2024-09-10T03:14:39.468 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi164.front.sepia.ceph.com' 2024-09-10T03:14:39.468 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-505.el9.x86_64"... 2024-09-10T03:14:39.468 DEBUG:teuthology.orchestra.run.smithi164:> uname -r 2024-09-10T03:14:39.484 INFO:teuthology.orchestra.run.smithi164.stdout:5.14.0-505.el9.x86_64 2024-09-10T03:14:39.484 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-10T03:14:39.484 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-10T03:14:39.484 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-10T03:14:40.485 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-10T03:14:40.486 DEBUG:teuthology.orchestra.run.smithi164:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-10T03:14:40.562 INFO:teuthology.orchestra.run.smithi164.stdout:ttyS1 2024-09-10T03:14:40.584 DEBUG:teuthology.parallel:result is None 2024-09-10T03:14:40.584 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-10T03:14:40.591 INFO:teuthology.task.internal:Creating test directory... 2024-09-10T03:14:40.592 DEBUG:teuthology.orchestra.run.smithi130:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-10T03:14:40.595 DEBUG:teuthology.orchestra.run.smithi164:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-10T03:14:40.612 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-10T03:14:40.618 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-10T03:14:40.657 INFO:teuthology.task.internal:Creating archive directory... 2024-09-10T03:14:40.657 DEBUG:teuthology.orchestra.run.smithi130:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-10T03:14:40.660 DEBUG:teuthology.orchestra.run.smithi164:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-10T03:14:40.690 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-10T03:14:40.696 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-10T03:14:40.696 DEBUG:teuthology.orchestra.run.smithi130:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-10T03:14:40.729 DEBUG:teuthology.orchestra.run.smithi164:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-10T03:14:40.768 INFO:teuthology.orchestra.run.smithi130.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-10T03:14:40.768 INFO:teuthology.orchestra.run.smithi164.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-10T03:14:40.779 INFO:teuthology.orchestra.run.smithi130.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-10T03:14:40.780 INFO:teuthology.orchestra.run.smithi164.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-10T03:14:40.782 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-10T03:14:40.789 INFO:teuthology.task.internal:Configuring sudo... 2024-09-10T03:14:40.789 DEBUG:teuthology.orchestra.run.smithi130:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-10T03:14:40.824 DEBUG:teuthology.orchestra.run.smithi164:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-10T03:14:40.851 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-10T03:14:40.860 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-10T03:14:40.860 DEBUG:teuthology.orchestra.run.smithi130:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-10T03:14:40.892 DEBUG:teuthology.orchestra.run.smithi164:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-10T03:14:40.907 DEBUG:teuthology.orchestra.run.smithi130:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-10T03:14:40.991 DEBUG:teuthology.orchestra.run.smithi130:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-10T03:14:41.035 DEBUG:teuthology.orchestra.run.smithi130:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-10T03:14:41.120 DEBUG:teuthology.orchestra.run.smithi130:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-10T03:14:41.161 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-09-10T03:14:41.161 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-10T03:14:41.228 DEBUG:teuthology.orchestra.run.smithi164:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-10T03:14:41.263 DEBUG:teuthology.orchestra.run.smithi164:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-10T03:14:41.331 DEBUG:teuthology.orchestra.run.smithi164:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-10T03:14:41.414 DEBUG:teuthology.orchestra.run.smithi164:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-10T03:14:41.454 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-10T03:14:41.454 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-10T03:14:41.520 DEBUG:teuthology.orchestra.run.smithi130:> sudo service rsyslog restart 2024-09-10T03:14:41.522 DEBUG:teuthology.orchestra.run.smithi164:> sudo service rsyslog restart 2024-09-10T03:14:41.565 INFO:teuthology.orchestra.run.smithi130.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-10T03:14:41.597 INFO:teuthology.orchestra.run.smithi164.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-10T03:14:42.055 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-10T03:14:42.063 INFO:teuthology.task.internal:Starting timer... 2024-09-10T03:14:42.063 INFO:teuthology.run_tasks:Running task pcp... 2024-09-10T03:14:42.073 INFO:teuthology.run_tasks:Running task selinux... 2024-09-10T03:14:42.081 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-10T03:14:42.081 DEBUG:teuthology.orchestra.run.smithi130:> sudo service auditd rotate 2024-09-10T03:14:42.146 INFO:teuthology.orchestra.run.smithi130.stdout:Rotating logs: 2024-09-10T03:14:42.148 DEBUG:teuthology.orchestra.run.smithi164:> sudo service auditd rotate 2024-09-10T03:14:42.214 INFO:teuthology.orchestra.run.smithi164.stdout:Rotating logs: 2024-09-10T03:14:42.216 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-10T03:14:42.216 DEBUG:teuthology.orchestra.run.smithi130:> /usr/sbin/getenforce 2024-09-10T03:14:42.242 INFO:teuthology.orchestra.run.smithi130.stdout:Permissive 2024-09-10T03:14:42.243 DEBUG:teuthology.orchestra.run.smithi164:> /usr/sbin/getenforce 2024-09-10T03:14:42.273 INFO:teuthology.orchestra.run.smithi164.stdout:Permissive 2024-09-10T03:14:42.273 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi130.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi164.front.sepia.ceph.com': 'permissive'} 2024-09-10T03:14:42.273 DEBUG:teuthology.orchestra.run.smithi130:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-10T03:14:42.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-10T03:14:42.310 DEBUG:teuthology.orchestra.run.smithi164:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-10T03:14:42.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-10T03:14:42.339 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-10T03:14:42.339 DEBUG:teuthology.orchestra.run.smithi130:> sudo /usr/sbin/setenforce permissive 2024-09-10T03:14:42.384 DEBUG:teuthology.orchestra.run.smithi164:> sudo /usr/sbin/setenforce permissive 2024-09-10T03:14:42.414 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-10T03:14:42.423 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-10T03:14:42.435 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-10T03:14:42.436 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi130.front.sepia.ceph.com,smithi164.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-10T03:18:25.908 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi130.front.sepia.ceph.com'), Remote(name='ubuntu@smithi164.front.sepia.ceph.com')] 2024-09-10T03:18:25.909 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi130.front.sepia.ceph.com' 2024-09-10T03:18:25.911 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-10T03:18:25.983 DEBUG:teuthology.orchestra.run.smithi130:> true 2024-09-10T03:18:26.063 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi130.front.sepia.ceph.com' 2024-09-10T03:18:26.063 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi164.front.sepia.ceph.com' 2024-09-10T03:18:26.064 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-10T03:18:26.149 DEBUG:teuthology.orchestra.run.smithi164:> true 2024-09-10T03:18:26.230 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi164.front.sepia.ceph.com' 2024-09-10T03:18:26.230 INFO:teuthology.run_tasks:Running task clock... 2024-09-10T03:18:26.240 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-10T03:18:26.241 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-10T03:18:26.241 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-10T03:18:26.243 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-10T03:18:26.244 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-10T03:18:26.270 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-10T03:18:26.284 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-10T03:18:26.302 INFO:teuthology.orchestra.run.smithi164.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-10T03:18:26.308 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: ntpd: command not found 2024-09-10T03:18:26.317 INFO:teuthology.orchestra.run.smithi164.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-10T03:18:26.319 INFO:teuthology.orchestra.run.smithi130.stdout:506 Cannot talk to daemon 2024-09-10T03:18:26.334 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-10T03:18:26.343 INFO:teuthology.orchestra.run.smithi164.stderr:sudo: ntpd: command not found 2024-09-10T03:18:26.348 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-10T03:18:26.355 INFO:teuthology.orchestra.run.smithi164.stdout:506 Cannot talk to daemon 2024-09-10T03:18:26.370 INFO:teuthology.orchestra.run.smithi164.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-10T03:18:26.384 INFO:teuthology.orchestra.run.smithi164.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-10T03:18:26.407 INFO:teuthology.orchestra.run.smithi130.stderr:bash: line 1: ntpq: command not found 2024-09-10T03:18:26.411 INFO:teuthology.orchestra.run.smithi130.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-10T03:18:26.411 INFO:teuthology.orchestra.run.smithi130.stdout:=============================================================================== 2024-09-10T03:18:26.411 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-10T03:18:26.411 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-10T03:18:26.411 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-10T03:18:26.411 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-10T03:18:26.441 INFO:teuthology.orchestra.run.smithi164.stderr:bash: line 1: ntpq: command not found 2024-09-10T03:18:26.444 INFO:teuthology.orchestra.run.smithi164.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-10T03:18:26.444 INFO:teuthology.orchestra.run.smithi164.stdout:=============================================================================== 2024-09-10T03:18:26.444 INFO:teuthology.orchestra.run.smithi164.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-10T03:18:26.444 INFO:teuthology.orchestra.run.smithi164.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-10T03:18:26.445 INFO:teuthology.orchestra.run.smithi164.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-10T03:18:26.445 INFO:teuthology.orchestra.run.smithi164.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-10T03:18:26.446 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-10T03:18:26.540 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': '3a5e3f16720f432aea965c40a8f6eb480095ec6c'} 2024-09-10T03:18:26.540 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-10T03:18:26.541 INFO:tasks.cephadm:Cluster fsid is 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:18:26.541 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-10T03:18:26.541 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-10T03:18:26.541 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi130': '172.21.15.130', 'mon.smithi164': '172.21.15.164'} 2024-09-10T03:18:26.542 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-10T03:18:26.542 DEBUG:teuthology.orchestra.run.smithi130:> sudo hostname $(hostname -s) 2024-09-10T03:18:26.567 DEBUG:teuthology.orchestra.run.smithi164:> sudo hostname $(hostname -s) 2024-09-10T03:18:26.594 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-10T03:18:26.595 DEBUG:teuthology.orchestra.run.smithi130:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-10T03:18:26.815 INFO:teuthology.orchestra.run.smithi130.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 10 03:18 /home/ubuntu/cephtest/cephadm 2024-09-10T03:18:26.816 DEBUG:teuthology.orchestra.run.smithi164:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-10T03:18:26.949 INFO:teuthology.orchestra.run.smithi164.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 10 03:18 /home/ubuntu/cephtest/cephadm 2024-09-10T03:18:26.949 DEBUG:teuthology.orchestra.run.smithi130:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-10T03:18:26.967 DEBUG:teuthology.orchestra.run.smithi164:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-10T03:18:26.991 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-10T03:18:26.991 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-10T03:18:27.009 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-10T03:18:28.386 INFO:teuthology.orchestra.run.smithi164.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-10T03:18:28.650 INFO:teuthology.orchestra.run.smithi130.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-10T03:18:40.345 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2024-09-10T03:18:40.345 INFO:teuthology.orchestra.run.smithi164.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-10T03:18:40.345 INFO:teuthology.orchestra.run.smithi164.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-10T03:18:40.345 INFO:teuthology.orchestra.run.smithi164.stdout: "repo_digests": [ 2024-09-10T03:18:40.345 INFO:teuthology.orchestra.run.smithi164.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-10T03:18:40.345 INFO:teuthology.orchestra.run.smithi164.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-10T03:18:40.345 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2024-09-10T03:18:40.345 INFO:teuthology.orchestra.run.smithi164.stdout:} 2024-09-10T03:18:40.392 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:18:40.392 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-10T03:18:40.392 INFO:teuthology.orchestra.run.smithi130.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-10T03:18:40.393 INFO:teuthology.orchestra.run.smithi130.stdout: "repo_digests": [ 2024-09-10T03:18:40.393 INFO:teuthology.orchestra.run.smithi130.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-10T03:18:40.393 INFO:teuthology.orchestra.run.smithi130.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-10T03:18:40.393 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-09-10T03:18:40.393 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:18:40.410 DEBUG:teuthology.orchestra.run.smithi130:> sudo mkdir -p /etc/ceph 2024-09-10T03:18:40.471 DEBUG:teuthology.orchestra.run.smithi164:> sudo mkdir -p /etc/ceph 2024-09-10T03:18:40.500 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod 777 /etc/ceph 2024-09-10T03:18:40.525 DEBUG:teuthology.orchestra.run.smithi164:> sudo chmod 777 /etc/ceph 2024-09-10T03:18:40.565 INFO:tasks.cephadm:Writing seed config... 2024-09-10T03:18:40.566 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-10T03:18:40.566 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-10T03:18:40.566 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-10T03:18:40.566 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-10T03:18:40.567 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-10T03:18:40.567 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-10T03:18:40.567 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-10T03:18:40.567 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-10T03:18:40.567 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-09-10T03:18:40.567 DEBUG:teuthology.orchestra.run.smithi130:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-10T03:18:40.583 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 5829498e-6f23-11ef-bcea-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-10T03:18:40.583 DEBUG:teuthology.orchestra.run.smithi130:mon.smithi130> sudo journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@mon.smithi130.service 2024-09-10T03:18:40.626 INFO:tasks.cephadm:Bootstrapping... 2024-09-10T03:18:40.626 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 5829498e-6f23-11ef-bcea-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.130 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-10T03:18:40.816 INFO:teuthology.orchestra.run.smithi130.stderr:-------------------------------------------------------------------------------- 2024-09-10T03:18:40.816 INFO:teuthology.orchestra.run.smithi130.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '5829498e-6f23-11ef-bcea-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.130', '--skip-admin-label'] 2024-09-10T03:18:40.837 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: 5.2.2 2024-09-10T03:18:40.839 INFO:teuthology.orchestra.run.smithi130.stderr:Verifying podman|docker is present... 2024-09-10T03:18:40.866 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: 5.2.2 2024-09-10T03:18:40.869 INFO:teuthology.orchestra.run.smithi130.stderr:Verifying lvm2 is present... 2024-09-10T03:18:40.869 INFO:teuthology.orchestra.run.smithi130.stderr:Verifying time synchronization is in place... 2024-09-10T03:18:40.875 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-10T03:18:40.882 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: inactive 2024-09-10T03:18:40.889 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: enabled 2024-09-10T03:18:40.895 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: active 2024-09-10T03:18:40.895 INFO:teuthology.orchestra.run.smithi130.stderr:Unit chronyd.service is enabled and running 2024-09-10T03:18:40.895 INFO:teuthology.orchestra.run.smithi130.stderr:Repeating the final host check... 2024-09-10T03:18:40.915 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: 5.2.2 2024-09-10T03:18:40.917 INFO:teuthology.orchestra.run.smithi130.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-10T03:18:40.917 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl is present 2024-09-10T03:18:40.917 INFO:teuthology.orchestra.run.smithi130.stderr:lvcreate is present 2024-09-10T03:18:40.923 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-10T03:18:40.929 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: inactive 2024-09-10T03:18:40.936 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: enabled 2024-09-10T03:18:40.942 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: active 2024-09-10T03:18:40.943 INFO:teuthology.orchestra.run.smithi130.stderr:Unit chronyd.service is enabled and running 2024-09-10T03:18:40.943 INFO:teuthology.orchestra.run.smithi130.stderr:Host looks OK 2024-09-10T03:18:40.943 INFO:teuthology.orchestra.run.smithi130.stderr:Cluster fsid: 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:18:40.944 INFO:teuthology.orchestra.run.smithi130.stderr:Acquiring lock 139649535035280 on /run/cephadm/5829498e-6f23-11ef-bcea-c7b262605968.lock 2024-09-10T03:18:40.944 INFO:teuthology.orchestra.run.smithi130.stderr:Lock 139649535035280 acquired on /run/cephadm/5829498e-6f23-11ef-bcea-c7b262605968.lock 2024-09-10T03:18:40.944 INFO:teuthology.orchestra.run.smithi130.stderr:Verifying IP 172.21.15.130 port 3300 ... 2024-09-10T03:18:40.944 INFO:teuthology.orchestra.run.smithi130.stderr:Verifying IP 172.21.15.130 port 6789 ... 2024-09-10T03:18:40.944 INFO:teuthology.orchestra.run.smithi130.stderr:Base mon IP is 172.21.15.130, final addrv is [v2:172.21.15.130:3300,v1:172.21.15.130:6789] 2024-09-10T03:18:40.946 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.130 metric 100 2024-09-10T03:18:40.946 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.130 metric 100 2024-09-10T03:18:40.949 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-10T03:18:40.949 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-09-10T03:18:40.949 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-09-10T03:18:40.952 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-10T03:18:40.952 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-10T03:18:40.952 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-10T03:18:40.952 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-09-10T03:18:40.952 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe88:7f37/64 scope link noprefixroute 2024-09-10T03:18:40.952 INFO:teuthology.orchestra.run.smithi130.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-10T03:18:40.953 INFO:teuthology.orchestra.run.smithi130.stderr:Mon IP `172.21.15.130` is in CIDR network `172.21.0.0/20` 2024-09-10T03:18:40.953 INFO:teuthology.orchestra.run.smithi130.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-10T03:18:40.953 INFO:teuthology.orchestra.run.smithi130.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-10T03:18:41.053 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-10T03:18:41.419 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: Getting image source signatures 2024-09-10T03:18:41.420 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-10T03:18:41.420 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-10T03:18:41.420 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-10T03:18:41.420 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-10T03:18:41.420 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-10T03:18:41.420 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-10T03:18:41.588 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: Writing manifest to image destination 2024-09-10T03:18:41.690 INFO:teuthology.orchestra.run.smithi130.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-10T03:18:42.086 INFO:teuthology.orchestra.run.smithi130.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-10T03:18:42.648 INFO:teuthology.orchestra.run.smithi130.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-10T03:18:42.648 INFO:teuthology.orchestra.run.smithi130.stderr:Extracting ceph user uid/gid from container image... 2024-09-10T03:18:43.022 INFO:teuthology.orchestra.run.smithi130.stderr:stat: 167 167 2024-09-10T03:18:43.393 INFO:teuthology.orchestra.run.smithi130.stderr:Creating initial keys... 2024-09-10T03:18:43.736 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph-authtool: AQCTut9mjce1JhAAnPITqe1pcxvI4NSzIhxrzQ== 2024-09-10T03:18:44.657 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph-authtool: AQCUut9mUK5JIBAAtx2j+xDAhTvkRShu4iGFDg== 2024-09-10T03:18:45.537 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph-authtool: AQCVut9mrzHiGhAAGJxBQUkPg51gUp2fuUAirQ== 2024-09-10T03:18:46.143 INFO:teuthology.orchestra.run.smithi130.stderr:Creating initial monmap... 2024-09-10T03:18:46.525 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-10T03:18:46.526 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-10T03:18:46.526 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:18:46.526 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-10T03:18:47.058 INFO:teuthology.orchestra.run.smithi130.stderr:monmaptool for smithi130 [v2:172.21.15.130:3300,v1:172.21.15.130:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-10T03:18:47.059 INFO:teuthology.orchestra.run.smithi130.stderr:setting min_mon_release = octopus 2024-09-10T03:18:47.059 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/monmaptool: set fsid to 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:18:47.059 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-10T03:18:47.059 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-09-10T03:18:47.059 INFO:teuthology.orchestra.run.smithi130.stderr:Creating mon... 2024-09-10T03:18:47.980 INFO:teuthology.orchestra.run.smithi130.stderr:create mon.smithi130 on 2024-09-10T03:18:48.196 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-10T03:18:48.353 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-5829498e-6f23-11ef-bcea-c7b262605968.target → /etc/systemd/system/ceph-5829498e-6f23-11ef-bcea-c7b262605968.target. 2024-09-10T03:18:48.353 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-5829498e-6f23-11ef-bcea-c7b262605968.target → /etc/systemd/system/ceph-5829498e-6f23-11ef-bcea-c7b262605968.target. 2024-09-10T03:18:48.678 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: Failed to reset failed state of unit ceph-5829498e-6f23-11ef-bcea-c7b262605968@mon.smithi130.service: Unit ceph-5829498e-6f23-11ef-bcea-c7b262605968@mon.smithi130.service not loaded. 2024-09-10T03:18:48.686 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: Created symlink /etc/systemd/system/ceph-5829498e-6f23-11ef-bcea-c7b262605968.target.wants/ceph-5829498e-6f23-11ef-bcea-c7b262605968@mon.smithi130.service → /etc/systemd/system/ceph-5829498e-6f23-11ef-bcea-c7b262605968@.service. 2024-09-10T03:18:49.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:18:49 smithi130 podman[23252]: 2024-09-10 03:18:49.113779828 +0000 UTC m=+0.141937814 container create 17ee0ee982982e628d17c9b267f48b9b3eacd81272fd1f30fcf5fea7f1bbd03b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, vcs-type=git, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.buildah.version=1.19.8) 2024-09-10T03:18:50.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:18:50 smithi130 podman[23252]: 2024-09-10 03:18:50.328328222 +0000 UTC m=+1.356486209 container init 17ee0ee982982e628d17c9b267f48b9b3eacd81272fd1f30fcf5fea7f1bbd03b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130, name=centos-stream, vcs-type=git, version=8, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, release=754, distribution-scope=public, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, RELEASE=HEAD, vendor=Red Hat, Inc., architecture=x86_64) 2024-09-10T03:18:50.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:18:50 smithi130 podman[23252]: 2024-09-10 03:18:50.332786507 +0000 UTC m=+1.360944495 container start 17ee0ee982982e628d17c9b267f48b9b3eacd81272fd1f30fcf5fea7f1bbd03b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, vcs-type=git, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, name=centos-stream, io.openshift.tags=base centos centos-stream, ceph=True, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, release=754) 2024-09-10T03:18:50.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:18:50 smithi130 ceph-mon[23272]: mkfs 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:18:50.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:18:50 smithi130 ceph-mon[23272]: mon.smithi130 is new leader, mons smithi130 in quorum (ranks 0) 2024-09-10T03:18:51.277 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: disabled 2024-09-10T03:18:51.285 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: inactive 2024-09-10T03:18:51.286 INFO:teuthology.orchestra.run.smithi130.stderr:firewalld.service is not enabled 2024-09-10T03:18:51.286 INFO:teuthology.orchestra.run.smithi130.stderr:Not possible to enable service . firewalld.service is not available 2024-09-10T03:18:51.287 INFO:teuthology.orchestra.run.smithi130.stderr:Waiting for mon to start... 2024-09-10T03:18:51.287 INFO:teuthology.orchestra.run.smithi130.stderr:Waiting for mon... 2024-09-10T03:18:51.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:18:51 smithi130 bash[23252]: 17ee0ee982982e628d17c9b267f48b9b3eacd81272fd1f30fcf5fea7f1bbd03b 2024-09-10T03:18:51.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:18:51 smithi130 systemd[1]: Started Ceph mon.smithi130 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:18:51.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:18:51 smithi130 ceph-mon[23272]: mon.smithi130 is new leader, mons smithi130 in quorum (ranks 0) 2024-09-10T03:18:51.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:18:51 smithi130 ceph-mon[23272]: monmap e1: 1 mons at {smithi130=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0]} 2024-09-10T03:18:51.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:18:51 smithi130 ceph-mon[23272]: fsmap 2024-09-10T03:18:51.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:18:51 smithi130 ceph-mon[23272]: osdmap e1: 0 total, 0 up, 0 in 2024-09-10T03:18:51.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:18:51 smithi130 ceph-mon[23272]: mgrmap e1: no daemons active 2024-09-10T03:18:54.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:18:54 smithi130 ceph-mon[23272]: from='client.? 172.21.15.130:0/2635142039' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-10T03:18:55.060 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: cluster: 2024-09-10T03:18:55.060 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: id: 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:18:55.060 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-10T03:18:55.060 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-09-10T03:18:55.060 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: services: 2024-09-10T03:18:55.060 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi130 (age 3s) 2024-09-10T03:18:55.060 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-10T03:18:55.061 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-10T03:18:55.061 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-09-10T03:18:55.061 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: data: 2024-09-10T03:18:55.061 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-10T03:18:55.061 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-10T03:18:55.061 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-10T03:18:55.061 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: pgs: 2024-09-10T03:18:55.061 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-09-10T03:19:00.834 INFO:teuthology.orchestra.run.smithi130.stderr:mon is available 2024-09-10T03:19:00.835 INFO:teuthology.orchestra.run.smithi130.stderr:Assimilating anything we can from ceph.conf... 2024-09-10T03:19:01.634 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-09-10T03:19:01.635 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: [global] 2024-09-10T03:19:01.635 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: fsid = 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:19:01.635 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.130:3300,v1:172.21.15.130:6789] 2024-09-10T03:19:01.635 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-10T03:19:01.635 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-10T03:19:01.635 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-10T03:19:01.635 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-10T03:19:01.635 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-09-10T03:19:01.635 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: [mgr] 2024-09-10T03:19:01.635 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-10T03:19:01.636 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-09-10T03:19:01.636 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: [osd] 2024-09-10T03:19:01.636 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-10T03:19:01.636 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-10T03:19:01.988 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:01 smithi130 ceph-mon[23272]: from='client.? 172.21.15.130:0/39964316' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-10T03:19:01.988 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:01 smithi130 ceph-mon[23272]: from='client.? 172.21.15.130:0/39964316' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-10T03:19:02.274 INFO:teuthology.orchestra.run.smithi130.stderr:Generating new minimal ceph.conf... 2024-09-10T03:19:02.968 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:02 smithi130 ceph-mon[23272]: from='client.? 172.21.15.130:0/3364344823' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:19:03.222 INFO:teuthology.orchestra.run.smithi130.stderr:Restarting the monitor... 2024-09-10T03:19:03.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:03 smithi130 systemd[1]: Stopping Ceph mon.smithi130 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:19:03.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:03 smithi130 bash[23492]: Error: no container with name or ID "ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon.smithi130" found: no such container 2024-09-10T03:19:03.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:03 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130[23267]: 2024-09-10T03:19:03.396+0000 7f28674ca700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi130 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:19:03.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:03 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130[23267]: 2024-09-10T03:19:03.396+0000 7f28674ca700 -1 mon.smithi130@0(leader) e1 *** Got Signal Terminated *** 2024-09-10T03:19:03.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:03 smithi130 podman[23503]: 2024-09-10 03:19:03.459031115 +0000 UTC m=+0.182667924 container died 17ee0ee982982e628d17c9b267f48b9b3eacd81272fd1f30fcf5fea7f1bbd03b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, version=8, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, name=centos-stream, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_CLEAN=True, ceph=True, build-date=2022-05-03T08:36:31.336870, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., com.redhat.component=centos-stream-container) 2024-09-10T03:19:04.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:03 smithi130 podman[23503]: 2024-09-10 03:19:03.80491586 +0000 UTC m=+0.528552662 container cleanup 17ee0ee982982e628d17c9b267f48b9b3eacd81272fd1f30fcf5fea7f1bbd03b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, version=8, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, RELEASE=HEAD, release=754, name=centos-stream, GIT_CLEAN=True, vcs-type=git, io.openshift.expose-services=) 2024-09-10T03:19:04.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:03 smithi130 bash[23503]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130 2024-09-10T03:19:04.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:03 smithi130 bash[23530]: Error: no container with name or ID "ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon.smithi130" found: no such container 2024-09-10T03:19:04.404 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 podman[23515]: 2024-09-10 03:19:04.089162646 +0000 UTC m=+0.627714247 container remove 17ee0ee982982e628d17c9b267f48b9b3eacd81272fd1f30fcf5fea7f1bbd03b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130, release=754, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, GIT_CLEAN=True, version=8, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream) 2024-09-10T03:19:04.404 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@mon.smithi130.service: Deactivated successfully. 2024-09-10T03:19:04.404 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 systemd[1]: Stopped Ceph mon.smithi130 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:19:04.404 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 systemd[1]: Starting Ceph mon.smithi130 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:19:04.405 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 podman[23594]: 2024-09-10 03:19:04.306612948 +0000 UTC m=+0.106791182 container create f2b698da8d6e9e2b1300afb4ba700a2d2899100eac9ae16ddade4563f4b7527c (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130, ceph=True, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, RELEASE=HEAD, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, vcs-type=git, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , release=754, io.openshift.expose-services=, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, architecture=x86_64) 2024-09-10T03:19:04.405 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 podman[23594]: 2024-09-10 03:19:04.215983333 +0000 UTC m=+0.016161521 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-10T03:19:04.570 INFO:teuthology.orchestra.run.smithi130.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-10T03:19:04.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 podman[23594]: 2024-09-10 03:19:04.492161787 +0000 UTC m=+0.292339972 container init f2b698da8d6e9e2b1300afb4ba700a2d2899100eac9ae16ddade4563f4b7527c (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130, distribution-scope=public, release=754, io.k8s.display-name=CentOS Stream 8, vcs-type=git, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , GIT_CLEAN=True, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, version=8, build-date=2022-05-03T08:36:31.336870) 2024-09-10T03:19:04.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 podman[23594]: 2024-09-10 03:19:04.49542328 +0000 UTC m=+0.295601466 container start f2b698da8d6e9e2b1300afb4ba700a2d2899100eac9ae16ddade4563f4b7527c (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, vcs-type=git, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, version=8, architecture=x86_64, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, maintainer=Guillaume Abrioux , io.openshift.expose-services=, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-10T03:19:04.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: set uid:gid to 167:167 (ceph:ceph) 2024-09-10T03:19:04.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-10T03:19:04.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: pidfile_write: ignore empty --pid-file 2024-09-10T03:19:04.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: load: jerasure load: lrc 2024-09-10T03:19:04.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: RocksDB version: 6.15.5 2024-09-10T03:19:04.765 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-10T03:19:04.765 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Compile date Apr 18 2022 2024-09-10T03:19:04.766 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: DB SUMMARY 2024-09-10T03:19:04.766 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: DB Session ID: DT9B99N51LLDNTJG7XXM 2024-09-10T03:19:04.766 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: CURRENT file: CURRENT 2024-09-10T03:19:04.766 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: IDENTITY file: IDENTITY 2024-09-10T03:19:04.766 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-10T03:19:04.766 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi130/store.db dir, Total Num: 1, files: 000008.sst 2024-09-10T03:19:04.766 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi130/store.db: 000010.log size: 84022 ; 2024-09-10T03:19:04.766 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.error_if_exists: 0 2024-09-10T03:19:04.766 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.create_if_missing: 0 2024-09-10T03:19:04.766 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.paranoid_checks: 1 2024-09-10T03:19:04.766 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-10T03:19:04.766 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.env: 0x55da75ac6860 2024-09-10T03:19:04.766 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.fs: Posix File System 2024-09-10T03:19:04.767 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.info_log: 0x55da772a1fa0 2024-09-10T03:19:04.767 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_file_opening_threads: 16 2024-09-10T03:19:04.767 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.statistics: (nil) 2024-09-10T03:19:04.767 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.use_fsync: 0 2024-09-10T03:19:04.767 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_log_file_size: 0 2024-09-10T03:19:04.767 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-10T03:19:04.768 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-10T03:19:04.768 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.keep_log_file_num: 1000 2024-09-10T03:19:04.768 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.recycle_log_file_num: 0 2024-09-10T03:19:04.768 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.allow_fallocate: 1 2024-09-10T03:19:04.768 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.allow_mmap_reads: 0 2024-09-10T03:19:04.768 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.allow_mmap_writes: 0 2024-09-10T03:19:04.768 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.use_direct_reads: 0 2024-09-10T03:19:04.768 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-10T03:19:04.768 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.create_missing_column_families: 0 2024-09-10T03:19:04.768 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.db_log_dir: 2024-09-10T03:19:04.768 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi130/store.db 2024-09-10T03:19:04.768 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-10T03:19:04.768 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-10T03:19:04.769 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-10T03:19:04.769 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-10T03:19:04.769 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-10T03:19:04.769 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-10T03:19:04.769 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.advise_random_on_open: 1 2024-09-10T03:19:04.769 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.db_write_buffer_size: 0 2024-09-10T03:19:04.769 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.write_buffer_manager: 0x55da773922d0 2024-09-10T03:19:04.769 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-10T03:19:04.769 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-10T03:19:04.769 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-10T03:19:04.769 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-10T03:19:04.770 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.rate_limiter: (nil) 2024-09-10T03:19:04.770 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-10T03:19:04.770 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.wal_recovery_mode: 2 2024-09-10T03:19:04.770 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.enable_thread_tracking: 0 2024-09-10T03:19:04.770 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.enable_pipelined_write: 0 2024-09-10T03:19:04.770 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.unordered_write: 0 2024-09-10T03:19:04.770 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-10T03:19:04.770 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-10T03:19:04.770 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-10T03:19:04.770 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-10T03:19:04.770 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.row_cache: None 2024-09-10T03:19:04.770 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.wal_filter: None 2024-09-10T03:19:04.771 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-10T03:19:04.771 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.allow_ingest_behind: 0 2024-09-10T03:19:04.771 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.preserve_deletes: 0 2024-09-10T03:19:04.771 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.two_write_queues: 0 2024-09-10T03:19:04.771 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.manual_wal_flush: 0 2024-09-10T03:19:04.771 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.atomic_flush: 0 2024-09-10T03:19:04.771 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-10T03:19:04.771 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-10T03:19:04.771 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-10T03:19:04.771 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.log_readahead_size: 0 2024-09-10T03:19:04.771 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-10T03:19:04.771 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.best_efforts_recovery: 0 2024-09-10T03:19:04.771 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-10T03:19:04.771 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-10T03:19:04.771 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.allow_data_in_errors: 0 2024-09-10T03:19:04.771 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.db_host_id: __hostname__ 2024-09-10T03:19:04.772 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_background_jobs: 2 2024-09-10T03:19:04.772 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_background_compactions: -1 2024-09-10T03:19:04.772 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_subcompactions: 1 2024-09-10T03:19:04.772 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-10T03:19:04.772 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-10T03:19:04.772 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-10T03:19:04.772 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_total_wal_size: 0 2024-09-10T03:19:04.772 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-10T03:19:04.772 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-10T03:19:04.772 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-10T03:19:04.773 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-10T03:19:04.773 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_open_files: -1 2024-09-10T03:19:04.773 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.bytes_per_sync: 0 2024-09-10T03:19:04.773 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-10T03:19:04.773 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-10T03:19:04.773 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compaction_readahead_size: 0 2024-09-10T03:19:04.773 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_background_flushes: -1 2024-09-10T03:19:04.773 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Compression algorithms supported: 2024-09-10T03:19:04.773 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-10T03:19:04.773 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: kZSTD supported: 0 2024-09-10T03:19:04.773 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: kXpressCompression supported: 0 2024-09-10T03:19:04.773 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: kLZ4HCCompression supported: 1 2024-09-10T03:19:04.773 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: kLZ4Compression supported: 1 2024-09-10T03:19:04.773 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: kBZip2Compression supported: 0 2024-09-10T03:19:04.773 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: kZlibCompression supported: 1 2024-09-10T03:19:04.774 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: kSnappyCompression supported: 1 2024-09-10T03:19:04.774 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-10T03:19:04.774 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi130/store.db/MANIFEST-000009 2024-09-10T03:19:04.774 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-10T03:19:04.774 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-10T03:19:04.774 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.merge_operator: 2024-09-10T03:19:04.774 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compaction_filter: None 2024-09-10T03:19:04.774 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compaction_filter_factory: None 2024-09-10T03:19:04.774 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.sst_partitioner_factory: None 2024-09-10T03:19:04.774 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-10T03:19:04.774 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-10T03:19:04.774 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55da7726dd00) 2024-09-10T03:19:04.774 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: cache_index_and_filter_blocks: 1 2024-09-10T03:19:04.774 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-10T03:19:04.774 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-10T03:19:04.775 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: pin_top_level_index_and_filter: 1 2024-09-10T03:19:04.775 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: index_type: 0 2024-09-10T03:19:04.775 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: data_block_index_type: 0 2024-09-10T03:19:04.775 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: index_shortening: 1 2024-09-10T03:19:04.775 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-10T03:19:04.775 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: hash_index_allow_collision: 1 2024-09-10T03:19:04.775 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: checksum: 1 2024-09-10T03:19:04.775 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: no_block_cache: 0 2024-09-10T03:19:04.775 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: block_cache: 0x55da772d8170 2024-09-10T03:19:04.775 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: block_cache_name: BinnedLRUCache 2024-09-10T03:19:04.775 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: block_cache_options: 2024-09-10T03:19:04.775 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: capacity : 536870912 2024-09-10T03:19:04.775 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: num_shard_bits : 4 2024-09-10T03:19:04.775 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: strict_capacity_limit : 0 2024-09-10T03:19:04.775 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: high_pri_pool_ratio: 0.000 2024-09-10T03:19:04.775 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: block_cache_compressed: (nil) 2024-09-10T03:19:04.775 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: persistent_cache: (nil) 2024-09-10T03:19:04.776 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: block_size: 4096 2024-09-10T03:19:04.776 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: block_size_deviation: 10 2024-09-10T03:19:04.776 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: block_restart_interval: 16 2024-09-10T03:19:04.776 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: index_block_restart_interval: 1 2024-09-10T03:19:04.776 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: metadata_block_size: 4096 2024-09-10T03:19:04.776 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: partition_filters: 0 2024-09-10T03:19:04.776 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: use_delta_encoding: 1 2024-09-10T03:19:04.776 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-10T03:19:04.776 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: whole_key_filtering: 1 2024-09-10T03:19:04.776 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: verify_compression: 0 2024-09-10T03:19:04.776 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: read_amp_bytes_per_bit: 0 2024-09-10T03:19:04.776 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: format_version: 4 2024-09-10T03:19:04.776 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: enable_index_compression: 1 2024-09-10T03:19:04.776 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: block_align: 0 2024-09-10T03:19:04.776 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.write_buffer_size: 33554432 2024-09-10T03:19:04.777 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_write_buffer_number: 2 2024-09-10T03:19:04.777 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compression: NoCompression 2024-09-10T03:19:04.777 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.bottommost_compression: Disabled 2024-09-10T03:19:04.777 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.prefix_extractor: nullptr 2024-09-10T03:19:04.777 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-10T03:19:04.777 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.num_levels: 7 2024-09-10T03:19:04.777 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-10T03:19:04.777 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-10T03:19:04.777 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-10T03:19:04.777 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-10T03:19:04.777 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-10T03:19:04.777 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-10T03:19:04.777 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-10T03:19:04.777 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-10T03:19:04.777 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-10T03:19:04.778 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-10T03:19:04.778 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-10T03:19:04.778 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compression_opts.level: 32767 2024-09-10T03:19:04.778 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compression_opts.strategy: 0 2024-09-10T03:19:04.778 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-10T03:19:04.778 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-10T03:19:04.778 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-10T03:19:04.778 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compression_opts.enabled: false 2024-09-10T03:19:04.778 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-10T03:19:04.778 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-10T03:19:04.778 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-10T03:19:04.778 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.target_file_size_base: 67108864 2024-09-10T03:19:04.778 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-10T03:19:04.778 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-10T03:19:04.779 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-10T03:19:04.779 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-10T03:19:04.779 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-10T03:19:04.779 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-10T03:19:04.779 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-10T03:19:04.779 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-10T03:19:04.779 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-10T03:19:04.779 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-10T03:19:04.779 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-10T03:19:04.779 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-10T03:19:04.779 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-10T03:19:04.779 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.arena_block_size: 4194304 2024-09-10T03:19:04.779 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-10T03:19:04.779 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-10T03:19:04.779 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-10T03:19:04.780 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.disable_auto_compactions: 0 2024-09-10T03:19:04.780 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-10T03:19:04.780 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-10T03:19:04.780 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-10T03:19:04.780 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-10T03:19:04.780 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-10T03:19:04.780 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-10T03:19:04.780 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-10T03:19:04.780 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-10T03:19:04.780 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-10T03:19:04.780 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-10T03:19:04.780 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.table_properties_collectors: 2024-09-10T03:19:04.780 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.inplace_update_support: 0 2024-09-10T03:19:04.781 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-10T03:19:04.781 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-10T03:19:04.781 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-10T03:19:04.781 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-10T03:19:04.781 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.bloom_locality: 0 2024-09-10T03:19:04.781 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.max_successive_merges: 0 2024-09-10T03:19:04.781 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-10T03:19:04.781 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.paranoid_file_checks: 0 2024-09-10T03:19:04.781 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.force_consistency_checks: 1 2024-09-10T03:19:04.781 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.report_bg_io_stats: 0 2024-09-10T03:19:04.781 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.ttl: 2592000 2024-09-10T03:19:04.781 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-10T03:19:04.781 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.enable_blob_files: false 2024-09-10T03:19:04.781 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.min_blob_size: 0 2024-09-10T03:19:04.782 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.blob_file_size: 268435456 2024-09-10T03:19:04.782 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-10T03:19:04.782 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-10T03:19:04.782 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-10T03:19:04.782 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi130/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-10T03:19:04.782 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-10T03:19:04.782 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-10T03:19:04.782 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725938344521515, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-10T03:19:04.782 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-10T03:19:04.782 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-10T03:19:04.783 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725938344522283, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80070, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78333, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9946, "raw_average_key_size": 46, "raw_value_size": 72589, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1725938344, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "22d0da89-8c83-45f7-a40a-8a3fa16b036a", "db_session_id": "DT9B99N51LLDNTJG7XXM"}} 2024-09-10T03:19:04.783 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-10T03:19:04.783 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725938344522647, "job": 1, "event": "recovery_finished"} 2024-09-10T03:19:04.783 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi130/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-10T03:19:04.783 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55da772bea80 2024-09-10T03:19:04.783 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: DB pointer 0x55da772ce000 2024-09-10T03:19:04.783 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-10T03:19:04.783 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-10T03:19:04.783 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: ** DB Stats ** 2024-09-10T03:19:04.783 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-10T03:19:04.783 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-10T03:19:04.783 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-10T03:19:04.783 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-10T03:19:04.783 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-10T03:19:04.783 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-10T03:19:04.784 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-10T03:19:04.784 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: 2024-09-10T03:19:04.784 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: ** Compaction Stats [default] ** 2024-09-10T03:19:04.784 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-10T03:19:04.784 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-10T03:19:04.784 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: L0 2/0 79.99 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 145.2 0.00 0.00 1 0.001 0 0 2024-09-10T03:19:04.784 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Sum 2/0 79.99 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 145.2 0.00 0.00 1 0.001 0 0 2024-09-10T03:19:04.784 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 145.2 0.00 0.00 1 0.001 0 0 2024-09-10T03:19:04.784 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: 2024-09-10T03:19:04.784 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: ** Compaction Stats [default] ** 2024-09-10T03:19:04.784 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-10T03:19:04.784 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-10T03:19:04.784 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 145.2 0.00 0.00 1 0.001 0 0 2024-09-10T03:19:04.784 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-10T03:19:04.784 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-10T03:19:04.785 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-10T03:19:04.785 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-10T03:19:04.785 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-10T03:19:04.785 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-10T03:19:04.785 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Cumulative compaction: 0.00 GB write, 24.84 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-10T03:19:04.785 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Interval compaction: 0.00 GB write, 24.84 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-10T03:19:04.785 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-10T03:19:04.785 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: 2024-09-10T03:19:04.785 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-10T03:19:04.785 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: 2024-09-10T03:19:04.785 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: ** Compaction Stats [default] ** 2024-09-10T03:19:04.785 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-10T03:19:04.785 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-10T03:19:04.785 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: L0 2/0 79.99 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 145.2 0.00 0.00 1 0.001 0 0 2024-09-10T03:19:04.786 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Sum 2/0 79.99 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 145.2 0.00 0.00 1 0.001 0 0 2024-09-10T03:19:04.786 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-10T03:19:04.786 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: 2024-09-10T03:19:04.786 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: ** Compaction Stats [default] ** 2024-09-10T03:19:04.786 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-10T03:19:04.786 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-10T03:19:04.786 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 145.2 0.00 0.00 1 0.001 0 0 2024-09-10T03:19:04.786 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-10T03:19:04.786 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-10T03:19:04.786 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-10T03:19:04.786 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-10T03:19:04.786 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-10T03:19:04.786 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-10T03:19:04.787 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Cumulative compaction: 0.00 GB write, 24.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-10T03:19:04.787 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-10T03:19:04.787 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-10T03:19:04.787 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: 2024-09-10T03:19:04.787 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-10T03:19:04.787 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: starting mon.smithi130 rank 0 at public addrs [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] at bind addrs [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi130 fsid 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:19:04.788 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: mon.smithi130@-1(???) e1 preinit fsid 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:19:04.788 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: mon.smithi130@-1(???).mds e1 new map 2024-09-10T03:19:04.788 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: mon.smithi130@-1(???).mds e1 print_map 2024-09-10T03:19:04.788 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: e1 2024-09-10T03:19:04.788 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-10T03:19:04.789 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-10T03:19:04.789 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: legacy client fscid: -1 2024-09-10T03:19:04.789 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: 2024-09-10T03:19:04.789 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: No filesystems configured 2024-09-10T03:19:04.789 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: mon.smithi130@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-10T03:19:04.789 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: mon.smithi130@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-10T03:19:04.789 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: mon.smithi130@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-10T03:19:04.789 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: mon.smithi130@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-10T03:19:04.789 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: mon.smithi130@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-10T03:19:04.790 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 bash[23594]: f2b698da8d6e9e2b1300afb4ba700a2d2899100eac9ae16ddade4563f4b7527c 2024-09-10T03:19:04.790 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 systemd[1]: Started Ceph mon.smithi130 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:19:04.790 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: mon.smithi130 is new leader, mons smithi130 in quorum (ranks 0) 2024-09-10T03:19:04.790 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: monmap e1: 1 mons at {smithi130=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0]} 2024-09-10T03:19:04.790 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: fsmap 2024-09-10T03:19:04.790 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: osdmap e1: 0 total, 0 up, 0 in 2024-09-10T03:19:04.790 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:04 smithi130 ceph-mon[23614]: mgrmap e1: no daemons active 2024-09-10T03:19:05.602 INFO:teuthology.orchestra.run.smithi130.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-10T03:19:05.602 INFO:teuthology.orchestra.run.smithi130.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-10T03:19:05.602 INFO:teuthology.orchestra.run.smithi130.stderr:Creating mgr... 2024-09-10T03:19:05.603 INFO:teuthology.orchestra.run.smithi130.stderr:Verifying port 9283 ... 2024-09-10T03:19:05.765 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: Failed to reset failed state of unit ceph-5829498e-6f23-11ef-bcea-c7b262605968@mgr.smithi130.axcmel.service: Unit ceph-5829498e-6f23-11ef-bcea-c7b262605968@mgr.smithi130.axcmel.service not loaded. 2024-09-10T03:19:05.773 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: Created symlink /etc/systemd/system/ceph-5829498e-6f23-11ef-bcea-c7b262605968.target.wants/ceph-5829498e-6f23-11ef-bcea-c7b262605968@mgr.smithi130.axcmel.service → /etc/systemd/system/ceph-5829498e-6f23-11ef-bcea-c7b262605968@.service. 2024-09-10T03:19:06.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:06 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3974938243' entity='client.admin' 2024-09-10T03:19:06.456 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: disabled 2024-09-10T03:19:06.463 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: inactive 2024-09-10T03:19:06.464 INFO:teuthology.orchestra.run.smithi130.stderr:firewalld.service is not enabled 2024-09-10T03:19:06.465 INFO:teuthology.orchestra.run.smithi130.stderr:Not possible to enable service . firewalld.service is not available 2024-09-10T03:19:06.473 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: disabled 2024-09-10T03:19:06.480 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: inactive 2024-09-10T03:19:06.481 INFO:teuthology.orchestra.run.smithi130.stderr:firewalld.service is not enabled 2024-09-10T03:19:06.481 INFO:teuthology.orchestra.run.smithi130.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-10T03:19:06.481 INFO:teuthology.orchestra.run.smithi130.stderr:Waiting for mgr to start... 2024-09-10T03:19:06.481 INFO:teuthology.orchestra.run.smithi130.stderr:Waiting for mgr... 2024-09-10T03:19:07.557 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-09-10T03:19:07.557 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: { 2024-09-10T03:19:07.558 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "fsid": "5829498e-6f23-11ef-bcea-c7b262605968", 2024-09-10T03:19:07.558 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "health": { 2024-09-10T03:19:07.558 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-10T03:19:07.558 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "checks": {}, 2024-09-10T03:19:07.558 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "mutes": [] 2024-09-10T03:19:07.558 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:07.558 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-10T03:19:07.558 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum": [ 2024-09-10T03:19:07.558 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 0 2024-09-10T03:19:07.558 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-09-10T03:19:07.558 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-10T03:19:07.558 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "smithi130" 2024-09-10T03:19:07.558 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-09-10T03:19:07.559 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum_age": 3, 2024-09-10T03:19:07.559 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "monmap": { 2024-09-10T03:19:07.559 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-10T03:19:07.559 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-10T03:19:07.559 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-10T03:19:07.559 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:07.559 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osdmap": { 2024-09-10T03:19:07.559 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-10T03:19:07.559 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-10T03:19:07.559 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-10T03:19:07.559 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-10T03:19:07.559 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-10T03:19:07.563 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-10T03:19:07.563 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-10T03:19:07.563 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:07.563 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "pgmap": { 2024-09-10T03:19:07.563 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-10T03:19:07.563 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-10T03:19:07.563 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-10T03:19:07.563 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-10T03:19:07.563 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-10T03:19:07.563 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-10T03:19:07.564 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-10T03:19:07.564 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-10T03:19:07.564 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:07.564 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "fsmap": { 2024-09-10T03:19:07.564 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-10T03:19:07.564 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-10T03:19:07.564 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-10T03:19:07.564 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:07.564 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-10T03:19:07.564 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "available": false, 2024-09-10T03:19:07.564 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-10T03:19:07.564 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "modules": [ 2024-09-10T03:19:07.564 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "iostat", 2024-09-10T03:19:07.565 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "nfs", 2024-09-10T03:19:07.565 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "restful" 2024-09-10T03:19:07.565 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-09-10T03:19:07.565 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "services": {} 2024-09-10T03:19:07.565 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:07.565 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "servicemap": { 2024-09-10T03:19:07.565 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-10T03:19:07.565 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "modified": "2024-09-10T03:18:50.376229+0000", 2024-09-10T03:19:07.565 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "services": {} 2024-09-10T03:19:07.565 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:07.565 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-10T03:19:07.565 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: } 2024-09-10T03:19:07.870 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:07 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/2814799810' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-10T03:19:08.123 INFO:teuthology.orchestra.run.smithi130.stderr:mgr not available, waiting (1/15)... 2024-09-10T03:19:10.663 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-09-10T03:19:10.664 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: { 2024-09-10T03:19:10.664 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "fsid": "5829498e-6f23-11ef-bcea-c7b262605968", 2024-09-10T03:19:10.664 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "health": { 2024-09-10T03:19:10.664 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-10T03:19:10.664 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "checks": {}, 2024-09-10T03:19:10.664 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "mutes": [] 2024-09-10T03:19:10.664 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:10.664 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-10T03:19:10.664 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum": [ 2024-09-10T03:19:10.664 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 0 2024-09-10T03:19:10.665 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-09-10T03:19:10.665 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-10T03:19:10.665 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "smithi130" 2024-09-10T03:19:10.665 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-09-10T03:19:10.665 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-09-10T03:19:10.665 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "monmap": { 2024-09-10T03:19:10.665 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-10T03:19:10.665 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-10T03:19:10.665 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-10T03:19:10.665 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:10.665 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osdmap": { 2024-09-10T03:19:10.665 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-10T03:19:10.665 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-10T03:19:10.666 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-10T03:19:10.666 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-10T03:19:10.666 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-10T03:19:10.666 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-10T03:19:10.666 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-10T03:19:10.666 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:10.666 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "pgmap": { 2024-09-10T03:19:10.666 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-10T03:19:10.666 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-10T03:19:10.666 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-10T03:19:10.666 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-10T03:19:10.666 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-10T03:19:10.666 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-10T03:19:10.666 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-10T03:19:10.667 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-10T03:19:10.667 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:10.667 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "fsmap": { 2024-09-10T03:19:10.667 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-10T03:19:10.667 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-10T03:19:10.667 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-10T03:19:10.667 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:10.667 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-10T03:19:10.667 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "available": false, 2024-09-10T03:19:10.667 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-10T03:19:10.667 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "modules": [ 2024-09-10T03:19:10.667 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "iostat", 2024-09-10T03:19:10.667 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "nfs", 2024-09-10T03:19:10.667 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "restful" 2024-09-10T03:19:10.667 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-09-10T03:19:10.668 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "services": {} 2024-09-10T03:19:10.668 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:10.668 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "servicemap": { 2024-09-10T03:19:10.668 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-10T03:19:10.668 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "modified": "2024-09-10T03:18:50.376229+0000", 2024-09-10T03:19:10.668 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "services": {} 2024-09-10T03:19:10.668 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:10.668 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-10T03:19:10.668 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: } 2024-09-10T03:19:11.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:10 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/2877515757' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-10T03:19:11.278 INFO:teuthology.orchestra.run.smithi130.stderr:mgr not available, waiting (2/15)... 2024-09-10T03:19:12.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:12 smithi130 ceph-mon[23614]: Activating manager daemon smithi130.axcmel 2024-09-10T03:19:12.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:12 smithi130 ceph-mon[23614]: mgrmap e2: smithi130.axcmel(active, starting, since 0.00282777s) 2024-09-10T03:19:12.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:12 smithi130 ceph-mon[23614]: from='mgr.14100 172.21.15.130:0/1081114590' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-10T03:19:12.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:12 smithi130 ceph-mon[23614]: from='mgr.14100 172.21.15.130:0/1081114590' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-10T03:19:12.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:12 smithi130 ceph-mon[23614]: from='mgr.14100 172.21.15.130:0/1081114590' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-10T03:19:12.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:12 smithi130 ceph-mon[23614]: from='mgr.14100 172.21.15.130:0/1081114590' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi130"}]: dispatch 2024-09-10T03:19:12.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:12 smithi130 ceph-mon[23614]: from='mgr.14100 172.21.15.130:0/1081114590' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr metadata", "who": "smithi130.axcmel", "id": "smithi130.axcmel"}]: dispatch 2024-09-10T03:19:12.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:12 smithi130 ceph-mon[23614]: Manager daemon smithi130.axcmel is now available 2024-09-10T03:19:12.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:12 smithi130 ceph-mon[23614]: from='mgr.14100 172.21.15.130:0/1081114590' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:19:12.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:12 smithi130 ceph-mon[23614]: from='mgr.14100 172.21.15.130:0/1081114590' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:19:12.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:12 smithi130 ceph-mon[23614]: from='mgr.14100 172.21.15.130:0/1081114590' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:12.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:12 smithi130 ceph-mon[23614]: from='mgr.14100 172.21.15.130:0/1081114590' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:12.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:12 smithi130 ceph-mon[23614]: from='mgr.14100 172.21.15.130:0/1081114590' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:13.968 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-09-10T03:19:13.969 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: { 2024-09-10T03:19:13.969 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "fsid": "5829498e-6f23-11ef-bcea-c7b262605968", 2024-09-10T03:19:13.969 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "health": { 2024-09-10T03:19:13.969 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-10T03:19:13.969 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "checks": {}, 2024-09-10T03:19:13.969 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "mutes": [] 2024-09-10T03:19:13.969 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:13.969 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-10T03:19:13.969 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum": [ 2024-09-10T03:19:13.969 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 0 2024-09-10T03:19:13.969 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-09-10T03:19:13.969 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-10T03:19:13.969 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "smithi130" 2024-09-10T03:19:13.970 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-09-10T03:19:13.970 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-09-10T03:19:13.970 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "monmap": { 2024-09-10T03:19:13.970 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-10T03:19:13.970 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-10T03:19:13.970 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-10T03:19:13.970 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:13.970 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osdmap": { 2024-09-10T03:19:13.970 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-10T03:19:13.970 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-10T03:19:13.970 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-10T03:19:13.970 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-10T03:19:13.970 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-10T03:19:13.974 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-10T03:19:13.974 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-10T03:19:13.974 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:13.974 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "pgmap": { 2024-09-10T03:19:13.974 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-10T03:19:13.974 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-10T03:19:13.974 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-10T03:19:13.974 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-10T03:19:13.974 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-10T03:19:13.974 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-10T03:19:13.974 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-10T03:19:13.975 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-10T03:19:13.975 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:13.975 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "fsmap": { 2024-09-10T03:19:13.975 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-10T03:19:13.975 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-10T03:19:13.975 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-10T03:19:13.975 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:13.975 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-10T03:19:13.975 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "available": true, 2024-09-10T03:19:13.975 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-10T03:19:13.975 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "modules": [ 2024-09-10T03:19:13.975 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "iostat", 2024-09-10T03:19:13.975 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "nfs", 2024-09-10T03:19:13.975 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "restful" 2024-09-10T03:19:13.975 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ], 2024-09-10T03:19:13.976 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "services": {} 2024-09-10T03:19:13.976 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:13.976 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "servicemap": { 2024-09-10T03:19:13.976 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-10T03:19:13.976 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "modified": "2024-09-10T03:18:50.376229+0000", 2024-09-10T03:19:13.976 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "services": {} 2024-09-10T03:19:13.976 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: }, 2024-09-10T03:19:13.976 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-10T03:19:13.976 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: } 2024-09-10T03:19:14.589 INFO:teuthology.orchestra.run.smithi130.stderr:mgr is available 2024-09-10T03:19:14.707 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:14 smithi130 ceph-mon[23614]: mgrmap e3: smithi130.axcmel(active, since 1.00649s) 2024-09-10T03:19:14.707 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:14 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/1091305705' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-10T03:19:15.166 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-09-10T03:19:15.166 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: [global] 2024-09-10T03:19:15.166 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: fsid = 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:19:15.166 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-10T03:19:15.166 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-10T03:19:15.166 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-10T03:19:15.166 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-10T03:19:15.166 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-09-10T03:19:15.166 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: [mgr] 2024-09-10T03:19:15.166 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-10T03:19:15.166 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 2024-09-10T03:19:15.166 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: [osd] 2024-09-10T03:19:15.167 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-10T03:19:15.167 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-10T03:19:15.712 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:15 smithi130 ceph-mon[23614]: mgrmap e4: smithi130.axcmel(active, since 2s) 2024-09-10T03:19:15.712 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:15 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/460279221' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-10T03:19:15.974 INFO:teuthology.orchestra.run.smithi130.stderr:Enabling cephadm module... 2024-09-10T03:19:16.926 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:16 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/544333111' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-10T03:19:17.869 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:17 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/544333111' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-10T03:19:17.869 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:17 smithi130 ceph-mon[23614]: mgrmap e5: smithi130.axcmel(active, since 4s) 2024-09-10T03:19:17.882 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: { 2024-09-10T03:19:17.882 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-10T03:19:17.882 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "available": true, 2024-09-10T03:19:17.882 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "active_name": "smithi130.axcmel", 2024-09-10T03:19:17.883 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-10T03:19:17.883 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: } 2024-09-10T03:19:18.427 INFO:teuthology.orchestra.run.smithi130.stderr:Waiting for the mgr to restart... 2024-09-10T03:19:18.427 INFO:teuthology.orchestra.run.smithi130.stderr:Waiting for mgr epoch 5... 2024-09-10T03:19:18.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:18 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3288018175' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-10T03:19:23.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:22 smithi130 ceph-mon[23614]: Active manager daemon smithi130.axcmel restarted 2024-09-10T03:19:23.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:22 smithi130 ceph-mon[23614]: Activating manager daemon smithi130.axcmel 2024-09-10T03:19:23.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:22 smithi130 ceph-mon[23614]: osdmap e2: 0 total, 0 up, 0 in 2024-09-10T03:19:23.706 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: { 2024-09-10T03:19:23.706 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-10T03:19:23.706 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "initialized": true 2024-09-10T03:19:23.706 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: } 2024-09-10T03:19:24.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:23 smithi130 ceph-mon[23614]: mgrmap e6: smithi130.axcmel(active, starting, since 0.0528753s) 2024-09-10T03:19:24.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:23 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi130"}]: dispatch 2024-09-10T03:19:24.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:23 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr metadata", "who": "smithi130.axcmel", "id": "smithi130.axcmel"}]: dispatch 2024-09-10T03:19:24.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:23 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-10T03:19:24.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:23 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-10T03:19:24.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:23 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-10T03:19:24.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:23 smithi130 ceph-mon[23614]: Manager daemon smithi130.axcmel is now available 2024-09-10T03:19:24.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:23 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:24.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:23 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:24.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:23 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:19:24.014 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:23 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:19:24.014 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:23 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:19:24.014 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:23 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:19:24.014 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:23 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:19:24.014 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:23 smithi130 ceph-mon[23614]: [10/Sep/2024:03:19:23] ENGINE Bus STARTING 2024-09-10T03:19:24.257 INFO:teuthology.orchestra.run.smithi130.stderr:mgr epoch 5 is available 2024-09-10T03:19:24.258 INFO:teuthology.orchestra.run.smithi130.stderr:Setting orchestrator backend to cephadm... 2024-09-10T03:19:25.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:24 smithi130 ceph-mon[23614]: [10/Sep/2024:03:19:23] ENGINE Serving on https://172.21.15.130:7150 2024-09-10T03:19:25.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:24 smithi130 ceph-mon[23614]: [10/Sep/2024:03:19:23] ENGINE Bus STARTED 2024-09-10T03:19:25.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:24 smithi130 ceph-mon[23614]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-10T03:19:25.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:24 smithi130 ceph-mon[23614]: mgrmap e7: smithi130.axcmel(active, since 1.06003s) 2024-09-10T03:19:25.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:24 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:25.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:24 smithi130 ceph-mon[23614]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-10T03:19:25.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:24 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:19:26.088 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:25 smithi130 ceph-mon[23614]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:19:26.088 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:25 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:26.088 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:25 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:19:26.089 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:25 smithi130 ceph-mon[23614]: mgrmap e8: smithi130.axcmel(active, since 2s) 2024-09-10T03:19:26.089 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: value unchanged 2024-09-10T03:19:26.652 INFO:teuthology.orchestra.run.smithi130.stderr:Generating ssh key... 2024-09-10T03:19:26.953 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:26 smithi130 ceph-mon[23614]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:19:28.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:28 smithi130 ceph-mon[23614]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:19:28.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:28 smithi130 ceph-mon[23614]: Generating ssh key... 2024-09-10T03:19:28.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:28 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:28.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:28 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:28.525 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC9bcVnEjjO/MwWSrN1db/4+CwuMdqXmWdCy+LnAyLe2WGPe4l+PSs3i2TdmF3ZrLyS83prMuRWWoV/5sqX9FB0Jke9kdXBilVP7hlBIW+ugZCdnVIBlRIQoBnhxIsKB5Ar3541M2Gpqq2O58GQWbY9DFWmgnRIhOSi23Lmiu9X5RRx2AlI2FJDkyetCID/etDaQUBZUUBb0KdGN1sWOr99xyl+0mw7ccNyAWjtSp4rUXk8PHrJjD65o66WE/WPoleNnkUjnUn7N7MUfw70Balum1BBqtGAw9WyxjB6mcB+MSMb0y4cuFXHphDjaK0AIa+1yqRtuMcsjmQ/ikDiNiIGeXcRk8sEj+patcMKo0LFGB9mTakpdXxl/MAyvaQFsDrYpnCpyOTae0uSjS00z+AUzDSD7HHsseQI3jjaQQ064EVZFSFjadx0JsIth71oWBK8P8Uq/KUHgkBHT01u4V7Uq8fjCfAn5NdJM4QKqHLxfZXjSrQZvwYkIb1mTF5Rqa8= ceph-5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:19:29.181 INFO:teuthology.orchestra.run.smithi130.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-10T03:19:29.181 INFO:teuthology.orchestra.run.smithi130.stderr:Adding key to root@localhost authorized_keys... 2024-09-10T03:19:29.189 INFO:teuthology.orchestra.run.smithi130.stderr:Adding host smithi130... 2024-09-10T03:19:29.564 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:29 smithi130 ceph-mon[23614]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:19:30.631 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: Added host 'smithi130' with addr '172.21.15.130' 2024-09-10T03:19:31.204 INFO:teuthology.orchestra.run.smithi130.stderr:Deploying mon service with default placement... 2024-09-10T03:19:31.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:31 smithi130 ceph-mon[23614]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi130", "addr": "172.21.15.130", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:19:31.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:31 smithi130 ceph-mon[23614]: Deploying cephadm binary to smithi130 2024-09-10T03:19:31.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:31 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:31.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:31 smithi130 ceph-mon[23614]: Added host smithi130 2024-09-10T03:19:31.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:31 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:19:31.837 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-10T03:19:32.594 INFO:teuthology.orchestra.run.smithi130.stderr:Deploying mgr service with default placement... 2024-09-10T03:19:33.204 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:32 smithi130 ceph-mon[23614]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:19:33.205 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:32 smithi130 ceph-mon[23614]: Saving service mon spec with placement count:5 2024-09-10T03:19:33.205 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:32 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:33.248 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-10T03:19:34.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:34 smithi130 ceph-mon[23614]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:19:34.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:34 smithi130 ceph-mon[23614]: Saving service mgr spec with placement count:2 2024-09-10T03:19:34.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:34 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:34.656 INFO:teuthology.orchestra.run.smithi130.stderr:Deploying crash service with default placement... 2024-09-10T03:19:35.293 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-10T03:19:35.737 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:35 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:35.737 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:35 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:36.382 INFO:teuthology.orchestra.run.smithi130.stderr:Deploying prometheus service with default placement... 2024-09-10T03:19:36.421 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:36 smithi130 ceph-mon[23614]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:19:36.422 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:36 smithi130 ceph-mon[23614]: Saving service crash spec with placement * 2024-09-10T03:19:36.422 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:36 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:37.036 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-10T03:19:37.630 INFO:teuthology.orchestra.run.smithi130.stderr:Deploying grafana service with default placement... 2024-09-10T03:19:37.753 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:37 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:37.753 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:37 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:37.753 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:37 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:38.255 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-10T03:19:38.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:38 smithi130 ceph-mon[23614]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:19:38.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:38 smithi130 ceph-mon[23614]: Saving service prometheus spec with placement count:1 2024-09-10T03:19:38.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:38 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:39.626 INFO:teuthology.orchestra.run.smithi130.stderr:Deploying node-exporter service with default placement... 2024-09-10T03:19:39.749 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:39 smithi130 ceph-mon[23614]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:19:39.749 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:39 smithi130 ceph-mon[23614]: Saving service grafana spec with placement count:1 2024-09-10T03:19:40.301 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-10T03:19:40.883 INFO:teuthology.orchestra.run.smithi130.stderr:Deploying alertmanager service with default placement... 2024-09-10T03:19:41.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:41 smithi130 ceph-mon[23614]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:19:41.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:41 smithi130 ceph-mon[23614]: Saving service node-exporter spec with placement * 2024-09-10T03:19:41.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:41 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:41.512 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-10T03:19:42.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:42 smithi130 ceph-mon[23614]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:19:42.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:42 smithi130 ceph-mon[23614]: Saving service alertmanager spec with placement count:1 2024-09-10T03:19:42.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:42 smithi130 ceph-mon[23614]: from='mgr.14118 172.21.15.130:0/3215470910' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:43.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:43 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/597158824' entity='client.admin' 2024-09-10T03:19:43.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:43 smithi130 ceph-mon[23614]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:19:44.401 INFO:teuthology.orchestra.run.smithi130.stderr:Enabling the dashboard module... 2024-09-10T03:19:45.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:44 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/231022694' entity='client.admin' 2024-09-10T03:19:46.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:45 smithi130 ceph-mon[23614]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:19:46.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:45 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/1993625894' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-10T03:19:47.146 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:46 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/1993625894' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-10T03:19:47.146 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:46 smithi130 ceph-mon[23614]: mgrmap e9: smithi130.axcmel(active, since 23s) 2024-09-10T03:19:47.287 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: { 2024-09-10T03:19:47.287 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-10T03:19:47.287 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "available": true, 2024-09-10T03:19:47.287 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "active_name": "smithi130.axcmel", 2024-09-10T03:19:47.287 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-10T03:19:47.287 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: } 2024-09-10T03:19:47.872 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:47 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3129325093' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-10T03:19:47.874 INFO:teuthology.orchestra.run.smithi130.stderr:Waiting for the mgr to restart... 2024-09-10T03:19:47.874 INFO:teuthology.orchestra.run.smithi130.stderr:Waiting for mgr epoch 9... 2024-09-10T03:19:52.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:52 smithi130 ceph-mon[23614]: Active manager daemon smithi130.axcmel restarted 2024-09-10T03:19:52.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:52 smithi130 ceph-mon[23614]: Activating manager daemon smithi130.axcmel 2024-09-10T03:19:52.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:52 smithi130 ceph-mon[23614]: osdmap e3: 0 total, 0 up, 0 in 2024-09-10T03:19:53.087 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: { 2024-09-10T03:19:53.087 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-10T03:19:53.087 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: "initialized": true 2024-09-10T03:19:53.087 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: } 2024-09-10T03:19:53.130 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:53 smithi130 ceph-mon[23614]: mgrmap e10: smithi130.axcmel(active, starting, since 0.053562s) 2024-09-10T03:19:53.130 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:53 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi130"}]: dispatch 2024-09-10T03:19:53.130 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:53 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr metadata", "who": "smithi130.axcmel", "id": "smithi130.axcmel"}]: dispatch 2024-09-10T03:19:53.130 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:53 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-10T03:19:53.130 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:53 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-10T03:19:53.130 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:53 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-10T03:19:53.130 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:53 smithi130 ceph-mon[23614]: Manager daemon smithi130.axcmel is now available 2024-09-10T03:19:53.131 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:53 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:19:53.131 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:53 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:19:53.131 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:53 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:19:53.131 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:53 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:19:53.131 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:53 smithi130 ceph-mon[23614]: [10/Sep/2024:03:19:52] ENGINE Bus STARTING 2024-09-10T03:19:53.131 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:53 smithi130 ceph-mon[23614]: [10/Sep/2024:03:19:52] ENGINE Serving on https://172.21.15.130:7150 2024-09-10T03:19:53.131 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:53 smithi130 ceph-mon[23614]: [10/Sep/2024:03:19:52] ENGINE Bus STARTED 2024-09-10T03:19:53.131 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:53 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:53.693 INFO:teuthology.orchestra.run.smithi130.stderr:mgr epoch 9 is available 2024-09-10T03:19:53.694 INFO:teuthology.orchestra.run.smithi130.stderr:Generating a dashboard self-signed certificate... 2024-09-10T03:19:54.395 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:54 smithi130 ceph-mon[23614]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-10T03:19:54.395 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:54 smithi130 ceph-mon[23614]: mgrmap e11: smithi130.axcmel(active, since 1.06169s) 2024-09-10T03:19:54.395 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:54 smithi130 ceph-mon[23614]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-10T03:19:54.395 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-10T03:19:55.051 INFO:teuthology.orchestra.run.smithi130.stderr:Creating initial admin user... 2024-09-10T03:19:55.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:55 smithi130 ceph-mon[23614]: mgrmap e12: smithi130.axcmel(active, since 2s) 2024-09-10T03:19:55.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:55 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:55.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:55 smithi130 ceph-mon[23614]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:19:55.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:55 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:55.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:55 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:55.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:55 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:55.985 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$93u2JmJMDVjYfnPP3MBG2OA5HXlxUnTO.YRy1qBDI2CsZNcsOdfga", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1725938395, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-10T03:19:56.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:56 smithi130 ceph-mon[23614]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:19:56.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:56 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:19:57.240 INFO:teuthology.orchestra.run.smithi130.stderr:Fetching dashboard port number... 2024-09-10T03:19:57.875 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: 8443 2024-09-10T03:19:58.218 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:57 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/2909120609' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-10T03:19:58.532 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: disabled 2024-09-10T03:19:58.540 INFO:teuthology.orchestra.run.smithi130.stderr:systemctl: inactive 2024-09-10T03:19:58.540 INFO:teuthology.orchestra.run.smithi130.stderr:firewalld.service is not enabled 2024-09-10T03:19:58.541 INFO:teuthology.orchestra.run.smithi130.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-10T03:19:58.542 INFO:teuthology.orchestra.run.smithi130.stderr:Ceph Dashboard is now available at: 2024-09-10T03:19:58.543 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-09-10T03:19:58.543 INFO:teuthology.orchestra.run.smithi130.stderr: URL: https://smithi130.front.sepia.ceph.com:8443/ 2024-09-10T03:19:58.543 INFO:teuthology.orchestra.run.smithi130.stderr: User: admin 2024-09-10T03:19:58.543 INFO:teuthology.orchestra.run.smithi130.stderr: Password: sn3l1tg5xr 2024-09-10T03:19:58.543 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-09-10T03:19:58.543 INFO:teuthology.orchestra.run.smithi130.stderr:Enabling autotune for osd_memory_target 2024-09-10T03:20:00.146 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:19:59 smithi130 ceph-mon[23614]: mgrmap e13: smithi130.axcmel(active, since 6s) 2024-09-10T03:20:00.527 INFO:teuthology.orchestra.run.smithi130.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-10T03:20:01.145 INFO:teuthology.orchestra.run.smithi130.stderr:You can access the Ceph CLI with: 2024-09-10T03:20:01.145 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-09-10T03:20:01.145 INFO:teuthology.orchestra.run.smithi130.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-10T03:20:01.145 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-09-10T03:20:01.145 INFO:teuthology.orchestra.run.smithi130.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-10T03:20:01.145 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-09-10T03:20:01.145 INFO:teuthology.orchestra.run.smithi130.stderr: ceph telemetry on 2024-09-10T03:20:01.145 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-09-10T03:20:01.145 INFO:teuthology.orchestra.run.smithi130.stderr:For more information see: 2024-09-10T03:20:01.146 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-09-10T03:20:01.146 INFO:teuthology.orchestra.run.smithi130.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-10T03:20:01.146 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-09-10T03:20:01.146 INFO:teuthology.orchestra.run.smithi130.stderr:Bootstrap complete. 2024-09-10T03:20:01.184 INFO:tasks.cephadm:Fetching config... 2024-09-10T03:20:01.184 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-09-10T03:20:01.184 DEBUG:teuthology.orchestra.run.smithi130:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-10T03:20:01.200 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-10T03:20:01.201 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-09-10T03:20:01.201 DEBUG:teuthology.orchestra.run.smithi130:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-10T03:20:01.256 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-10T03:20:01.256 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-09-10T03:20:01.256 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/keyring of=/dev/stdout 2024-09-10T03:20:01.322 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-10T03:20:01.322 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-09-10T03:20:01.322 DEBUG:teuthology.orchestra.run.smithi130:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-10T03:20:01.378 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-10T03:20:01.378 DEBUG:teuthology.orchestra.run.smithi130:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC9bcVnEjjO/MwWSrN1db/4+CwuMdqXmWdCy+LnAyLe2WGPe4l+PSs3i2TdmF3ZrLyS83prMuRWWoV/5sqX9FB0Jke9kdXBilVP7hlBIW+ugZCdnVIBlRIQoBnhxIsKB5Ar3541M2Gpqq2O58GQWbY9DFWmgnRIhOSi23Lmiu9X5RRx2AlI2FJDkyetCID/etDaQUBZUUBb0KdGN1sWOr99xyl+0mw7ccNyAWjtSp4rUXk8PHrJjD65o66WE/WPoleNnkUjnUn7N7MUfw70Balum1BBqtGAw9WyxjB6mcB+MSMb0y4cuFXHphDjaK0AIa+1yqRtuMcsjmQ/ikDiNiIGeXcRk8sEj+patcMKo0LFGB9mTakpdXxl/MAyvaQFsDrYpnCpyOTae0uSjS00z+AUzDSD7HHsseQI3jjaQQ064EVZFSFjadx0JsIth71oWBK8P8Uq/KUHgkBHT01u4V7Uq8fjCfAn5NdJM4QKqHLxfZXjSrQZvwYkIb1mTF5Rqa8= ceph-5829498e-6f23-11ef-bcea-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-10T03:20:01.461 INFO:teuthology.orchestra.run.smithi130.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC9bcVnEjjO/MwWSrN1db/4+CwuMdqXmWdCy+LnAyLe2WGPe4l+PSs3i2TdmF3ZrLyS83prMuRWWoV/5sqX9FB0Jke9kdXBilVP7hlBIW+ugZCdnVIBlRIQoBnhxIsKB5Ar3541M2Gpqq2O58GQWbY9DFWmgnRIhOSi23Lmiu9X5RRx2AlI2FJDkyetCID/etDaQUBZUUBb0KdGN1sWOr99xyl+0mw7ccNyAWjtSp4rUXk8PHrJjD65o66WE/WPoleNnkUjnUn7N7MUfw70Balum1BBqtGAw9WyxjB6mcB+MSMb0y4cuFXHphDjaK0AIa+1yqRtuMcsjmQ/ikDiNiIGeXcRk8sEj+patcMKo0LFGB9mTakpdXxl/MAyvaQFsDrYpnCpyOTae0uSjS00z+AUzDSD7HHsseQI3jjaQQ064EVZFSFjadx0JsIth71oWBK8P8Uq/KUHgkBHT01u4V7Uq8fjCfAn5NdJM4QKqHLxfZXjSrQZvwYkIb1mTF5Rqa8= ceph-5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:20:01.472 DEBUG:teuthology.orchestra.run.smithi164:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC9bcVnEjjO/MwWSrN1db/4+CwuMdqXmWdCy+LnAyLe2WGPe4l+PSs3i2TdmF3ZrLyS83prMuRWWoV/5sqX9FB0Jke9kdXBilVP7hlBIW+ugZCdnVIBlRIQoBnhxIsKB5Ar3541M2Gpqq2O58GQWbY9DFWmgnRIhOSi23Lmiu9X5RRx2AlI2FJDkyetCID/etDaQUBZUUBb0KdGN1sWOr99xyl+0mw7ccNyAWjtSp4rUXk8PHrJjD65o66WE/WPoleNnkUjnUn7N7MUfw70Balum1BBqtGAw9WyxjB6mcB+MSMb0y4cuFXHphDjaK0AIa+1yqRtuMcsjmQ/ikDiNiIGeXcRk8sEj+patcMKo0LFGB9mTakpdXxl/MAyvaQFsDrYpnCpyOTae0uSjS00z+AUzDSD7HHsseQI3jjaQQ064EVZFSFjadx0JsIth71oWBK8P8Uq/KUHgkBHT01u4V7Uq8fjCfAn5NdJM4QKqHLxfZXjSrQZvwYkIb1mTF5Rqa8= ceph-5829498e-6f23-11ef-bcea-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-10T03:20:01.546 INFO:teuthology.orchestra.run.smithi164.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC9bcVnEjjO/MwWSrN1db/4+CwuMdqXmWdCy+LnAyLe2WGPe4l+PSs3i2TdmF3ZrLyS83prMuRWWoV/5sqX9FB0Jke9kdXBilVP7hlBIW+ugZCdnVIBlRIQoBnhxIsKB5Ar3541M2Gpqq2O58GQWbY9DFWmgnRIhOSi23Lmiu9X5RRx2AlI2FJDkyetCID/etDaQUBZUUBb0KdGN1sWOr99xyl+0mw7ccNyAWjtSp4rUXk8PHrJjD65o66WE/WPoleNnkUjnUn7N7MUfw70Balum1BBqtGAw9WyxjB6mcB+MSMb0y4cuFXHphDjaK0AIa+1yqRtuMcsjmQ/ikDiNiIGeXcRk8sEj+patcMKo0LFGB9mTakpdXxl/MAyvaQFsDrYpnCpyOTae0uSjS00z+AUzDSD7HHsseQI3jjaQQ064EVZFSFjadx0JsIth71oWBK8P8Uq/KUHgkBHT01u4V7Uq8fjCfAn5NdJM4QKqHLxfZXjSrQZvwYkIb1mTF5Rqa8= ceph-5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:20:01.559 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-10T03:20:01.562 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:01 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/254160971' entity='client.admin' 2024-09-10T03:20:05.721 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:05 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/868412992' entity='client.admin' 2024-09-10T03:20:05.722 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:05 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:05.722 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:05 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd/host:smithi130", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:20:05.722 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:05 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:05.744 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-10T03:20:05.745 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-10T03:20:06.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:06 smithi130 ceph-mon[23614]: Deploying daemon alertmanager.smithi130 on smithi130 2024-09-10T03:20:09.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:09 smithi130 ceph-mon[23614]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:20:09.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:09 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:09.621 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi164 2024-09-10T03:20:09.621 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-10T03:20:09.621 DEBUG:teuthology.orchestra.run.smithi164:> dd of=/etc/ceph/ceph.conf 2024-09-10T03:20:09.640 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-10T03:20:09.640 DEBUG:teuthology.orchestra.run.smithi164:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-10T03:20:09.698 INFO:tasks.cephadm:Adding host smithi164 to orchestrator... 2024-09-10T03:20:09.698 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph orch host add smithi164 2024-09-10T03:20:13.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:13 smithi130 ceph-mon[23614]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:20:13.976 INFO:teuthology.orchestra.run.smithi130.stdout:Added host 'smithi164' with addr '172.21.15.164' 2024-09-10T03:20:14.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:14 smithi130 ceph-mon[23614]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi164", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:20:14.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:14 smithi130 ceph-mon[23614]: Deploying cephadm binary to smithi164 2024-09-10T03:20:14.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:14 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:15.159 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph orch host ls --format=json 2024-09-10T03:20:15.680 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:15 smithi130 ceph-mon[23614]: Added host smithi164 2024-09-10T03:20:15.681 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:15 smithi130 ceph-mon[23614]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:20:17.239 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:16 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:17.240 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:16 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi130", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-10T03:20:17.240 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:16 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi130", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-10T03:20:17.240 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:16 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:20:17.240 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:16 smithi130 ceph-mon[23614]: Deploying daemon crash.smithi130 on smithi130 2024-09-10T03:20:17.240 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:16 smithi130 ceph-mon[23614]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:20:18.397 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:18 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:18.610 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:20:18.610 INFO:teuthology.orchestra.run.smithi130.stdout:[{"addr": "172.21.15.130", "hostname": "smithi130", "labels": [], "status": ""}, {"addr": "172.21.15.164", "hostname": "smithi164", "labels": [], "status": ""}] 2024-09-10T03:20:19.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:19 smithi130 ceph-mon[23614]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:20:19.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:19 smithi130 ceph-mon[23614]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-10T03:20:19.396 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-10T03:20:19.397 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph osd crush tunables default 2024-09-10T03:20:20.900 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:20 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:20.900 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:20 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:20.900 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:20 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:20.900 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:20 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-10T03:20:20.901 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:20 smithi130 ceph-mon[23614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-10T03:20:20.901 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:20 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:20.901 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:20 smithi130 ceph-mon[23614]: Deploying daemon grafana.smithi130 on smithi130 2024-09-10T03:20:21.834 INFO:teuthology.orchestra.run.smithi130.stderr:adjusted tunables profile to default 2024-09-10T03:20:22.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:21 smithi130 ceph-mon[23614]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:20:22.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:21 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/349461942' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-10T03:20:22.716 INFO:tasks.cephadm:Adding mon.smithi130 on smithi130 2024-09-10T03:20:22.716 INFO:tasks.cephadm:Adding mon.smithi164 on smithi164 2024-09-10T03:20:22.716 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph orch apply mon '2;smithi130:172.21.15.130=smithi130;smithi164:172.21.15.164=smithi164' 2024-09-10T03:20:23.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:22 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/349461942' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-10T03:20:23.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:22 smithi130 ceph-mon[23614]: osdmap e4: 0 total, 0 up, 0 in 2024-09-10T03:20:23.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:22 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:24.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:23 smithi130 ceph-mon[23614]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:20:24.409 INFO:teuthology.orchestra.run.smithi164.stdout:Scheduled mon update... 2024-09-10T03:20:24.989 DEBUG:teuthology.orchestra.run.smithi164:mon.smithi164> sudo journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@mon.smithi164.service 2024-09-10T03:20:24.991 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-10T03:20:24.991 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph mon dump -f json 2024-09-10T03:20:25.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:25 smithi130 ceph-mon[23614]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:20:25.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:25 smithi130 ceph-mon[23614]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi130:172.21.15.130=smithi130;smithi164:172.21.15.164=smithi164", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:20:25.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:25 smithi130 ceph-mon[23614]: Saving service mon spec with placement smithi130:172.21.15.130=smithi130;smithi164:172.21.15.164=smithi164;count:2 2024-09-10T03:20:25.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:25 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:26.870 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:20:26.870 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"5829498e-6f23-11ef-bcea-c7b262605968","modified":"2024-09-10T03:18:46.462733Z","created":"2024-09-10T03:18:46.462733Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi130","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-10T03:20:26.872 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-09-10T03:20:27.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:27 smithi130 ceph-mon[23614]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:20:27.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:27 smithi130 ceph-mon[23614]: from='client.? 172.21.15.164:0/885199302' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:20:28.485 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-10T03:20:28.485 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph mon dump -f json 2024-09-10T03:20:29.685 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:29 smithi130 ceph-mon[23614]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:20:30.305 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:20:30.305 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"5829498e-6f23-11ef-bcea-c7b262605968","modified":"2024-09-10T03:18:46.462733Z","created":"2024-09-10T03:18:46.462733Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi130","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-10T03:20:30.307 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-09-10T03:20:30.986 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:30 smithi130 ceph-mon[23614]: from='client.? 172.21.15.164:0/923517571' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:20:31.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:31 smithi130 ceph-mon[23614]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:20:31.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:31 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:31.816 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-10T03:20:31.816 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph mon dump -f json 2024-09-10T03:20:33.009 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:32 smithi130 ceph-mon[23614]: Deploying daemon node-exporter.smithi130 on smithi130 2024-09-10T03:20:33.009 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:32 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:33.612 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:20:33.612 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"5829498e-6f23-11ef-bcea-c7b262605968","modified":"2024-09-10T03:18:46.462733Z","created":"2024-09-10T03:18:46.462733Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi130","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-10T03:20:33.613 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-09-10T03:20:33.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:33 smithi130 ceph-mon[23614]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:20:35.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:34 smithi130 ceph-mon[23614]: from='client.? 172.21.15.164:0/2442967967' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:20:35.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:34 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:35.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:34 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-10T03:20:35.168 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-10T03:20:35.168 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph mon dump -f json 2024-09-10T03:20:35.906 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:35 smithi130 ceph-mon[23614]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:20:35.906 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:35 smithi130 ceph-mon[23614]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-10T03:20:35.906 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:35 smithi130 ceph-mon[23614]: mgrmap e14: smithi130.axcmel(active, since 43s) 2024-09-10T03:20:36.846 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:20:36.846 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"5829498e-6f23-11ef-bcea-c7b262605968","modified":"2024-09-10T03:18:46.462733Z","created":"2024-09-10T03:18:46.462733Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi130","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-10T03:20:36.848 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-09-10T03:20:37.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:36 smithi130 ceph-mon[23614]: from='client.? 172.21.15.164:0/3102429567' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:20:38.420 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-10T03:20:38.420 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph mon dump -f json 2024-09-10T03:20:40.202 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:20:40.202 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"5829498e-6f23-11ef-bcea-c7b262605968","modified":"2024-09-10T03:18:46.462733Z","created":"2024-09-10T03:18:46.462733Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi130","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-10T03:20:40.204 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-09-10T03:20:41.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:40 smithi130 ceph-mon[23614]: from='client.? 172.21.15.164:0/33711504' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:20:41.765 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-10T03:20:41.765 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph mon dump -f json 2024-09-10T03:20:42.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: Active manager daemon smithi130.axcmel restarted 2024-09-10T03:20:42.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: Activating manager daemon smithi130.axcmel 2024-09-10T03:20:42.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: osdmap e5: 0 total, 0 up, 0 in 2024-09-10T03:20:42.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: mgrmap e15: smithi130.axcmel(active, starting, since 0.183151s) 2024-09-10T03:20:42.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi130"}]: dispatch 2024-09-10T03:20:42.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr metadata", "who": "smithi130.axcmel", "id": "smithi130.axcmel"}]: dispatch 2024-09-10T03:20:42.764 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-10T03:20:42.764 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-10T03:20:42.764 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-10T03:20:42.764 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: Manager daemon smithi130.axcmel is now available 2024-09-10T03:20:42.764 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:20:42.764 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:20:42.764 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:20:42.764 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:20:42.764 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:42.764 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:20:42.764 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:20:42.765 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: [10/Sep/2024:03:20:41] ENGINE Bus STARTING 2024-09-10T03:20:42.765 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: [10/Sep/2024:03:20:41] ENGINE Serving on https://172.21.15.130:7150 2024-09-10T03:20:42.765 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: [10/Sep/2024:03:20:41] ENGINE Bus STARTED 2024-09-10T03:20:42.765 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:43.572 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:20:43.572 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"5829498e-6f23-11ef-bcea-c7b262605968","modified":"2024-09-10T03:18:46.462733Z","created":"2024-09-10T03:18:46.462733Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi130","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-10T03:20:43.574 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-09-10T03:20:43.918 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:43 smithi130 ceph-mon[23614]: mgrmap e16: smithi130.axcmel(active, since 1.18589s) 2024-09-10T03:20:43.919 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:43 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:43.919 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:43 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:44.822 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:44 smithi130 ceph-mon[23614]: from='client.? 172.21.15.164:0/4204399153' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:20:44.822 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:44 smithi130 ceph-mon[23614]: mgrmap e17: smithi130.axcmel(active, since 2s) 2024-09-10T03:20:44.822 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:44 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:45.074 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-10T03:20:45.074 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph mon dump -f json 2024-09-10T03:20:46.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:46 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:46.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:46 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:46.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:46 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd/host:smithi130", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:20:46.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:46 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:47.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:47 smithi130 ceph-mon[23614]: Updating smithi130:/etc/ceph/ceph.conf 2024-09-10T03:20:47.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:47 smithi130 ceph-mon[23614]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-09-10T03:20:47.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:47 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:47.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:47 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:47.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:47 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd/host:smithi164", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:20:47.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:47 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:47.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:47 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi164", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-10T03:20:47.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:47 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi164", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-10T03:20:47.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:47 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:20:48.041 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:20:48.041 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"5829498e-6f23-11ef-bcea-c7b262605968","modified":"2024-09-10T03:18:46.462733Z","created":"2024-09-10T03:18:46.462733Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi130","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-10T03:20:48.043 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-09-10T03:20:48.414 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:48 smithi130 ceph-mon[23614]: Deploying daemon crash.smithi164 on smithi164 2024-09-10T03:20:49.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:49 smithi130 ceph-mon[23614]: from='client.? 172.21.15.164:0/3411075316' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:20:49.573 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-10T03:20:49.573 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph mon dump -f json 2024-09-10T03:20:51.593 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:20:51.593 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"5829498e-6f23-11ef-bcea-c7b262605968","modified":"2024-09-10T03:18:46.462733Z","created":"2024-09-10T03:18:46.462733Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi130","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-10T03:20:51.595 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-09-10T03:20:52.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:51 smithi130 ceph-mon[23614]: from='client.? 172.21.15.164:0/738441756' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:20:53.462 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-10T03:20:53.462 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph mon dump -f json 2024-09-10T03:20:53.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:53 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:53.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:53 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.bkfwap", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-10T03:20:53.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:53 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.bkfwap", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-10T03:20:53.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:53 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-10T03:20:53.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:53 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:20:53.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:53 smithi130 ceph-mon[23614]: Deploying daemon mgr.smithi164.bkfwap on smithi164 2024-09-10T03:20:55.426 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:20:55.426 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"5829498e-6f23-11ef-bcea-c7b262605968","modified":"2024-09-10T03:18:46.462733Z","created":"2024-09-10T03:18:46.462733Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi130","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-10T03:20:55.427 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-09-10T03:20:56.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:55 smithi130 ceph-mon[23614]: from='client.? 172.21.15.164:0/357722886' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:20:57.243 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-10T03:20:57.243 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph mon dump -f json 2024-09-10T03:20:57.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:57.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-10T03:20:57.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:20:57.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:56 smithi130 ceph-mon[23614]: Deploying daemon mon.smithi164 on smithi164 2024-09-10T03:20:57.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:20:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:20:58.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:20:58 smithi164 systemd[1]: Starting Ceph mon.smithi164 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:20:59.870 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:20:59 smithi164 podman[25629]: 2024-09-10 03:20:59.503206812 +0000 UTC m=+0.546286305 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-10T03:20:59.870 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:20:59 smithi164 podman[25629]: 2024-09-10 03:20:59.78275549 +0000 UTC m=+0.825834971 container create 7a928b0b045c1c66777736a5fe091bdfe77eb7ab7e9b47ec99384fc8460e6318 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi164, version=8, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, vcs-type=git, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux ) 2024-09-10T03:20:59.950 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:20:59.950 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"5829498e-6f23-11ef-bcea-c7b262605968","modified":"2024-09-10T03:18:46.462733Z","created":"2024-09-10T03:18:46.462733Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi130","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-10T03:20:59.952 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2024-09-10T03:21:00.147 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 podman[25629]: 2024-09-10 03:21:00.027649748 +0000 UTC m=+1.070729246 container init 7a928b0b045c1c66777736a5fe091bdfe77eb7ab7e9b47ec99384fc8460e6318 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi164, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, name=centos-stream, release=754, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, RELEASE=HEAD, ceph=True, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, architecture=x86_64, vcs-type=git, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-10T03:21:00.147 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 podman[25629]: 2024-09-10 03:21:00.031309935 +0000 UTC m=+1.074389417 container start 7a928b0b045c1c66777736a5fe091bdfe77eb7ab7e9b47ec99384fc8460e6318 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi164, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, version=8, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.openshift.expose-services=, architecture=x86_64, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870) 2024-09-10T03:21:00.148 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: set uid:gid to 167:167 (ceph:ceph) 2024-09-10T03:21:00.148 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-10T03:21:00.148 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: pidfile_write: ignore empty --pid-file 2024-09-10T03:21:00.148 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: load: jerasure load: lrc 2024-09-10T03:21:00.148 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: RocksDB version: 6.15.5 2024-09-10T03:21:00.148 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-10T03:21:00.148 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Compile date Apr 18 2022 2024-09-10T03:21:00.148 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: DB SUMMARY 2024-09-10T03:21:00.148 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: DB Session ID: M348FH1U3K8D99GIRARQ 2024-09-10T03:21:00.149 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: CURRENT file: CURRENT 2024-09-10T03:21:00.149 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: IDENTITY file: IDENTITY 2024-09-10T03:21:00.149 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-10T03:21:00.149 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi164/store.db dir, Total Num: 0, files: 2024-09-10T03:21:00.149 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi164/store.db: 000004.log size: 511 ; 2024-09-10T03:21:00.149 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.error_if_exists: 0 2024-09-10T03:21:00.149 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.create_if_missing: 0 2024-09-10T03:21:00.149 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.paranoid_checks: 1 2024-09-10T03:21:00.149 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-10T03:21:00.149 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.env: 0x55bf68cc6860 2024-09-10T03:21:00.149 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.fs: Posix File System 2024-09-10T03:21:00.149 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.info_log: 0x55bf6b4dffa0 2024-09-10T03:21:00.149 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_file_opening_threads: 16 2024-09-10T03:21:00.149 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.statistics: (nil) 2024-09-10T03:21:00.150 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.use_fsync: 0 2024-09-10T03:21:00.150 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_log_file_size: 0 2024-09-10T03:21:00.150 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-10T03:21:00.150 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-10T03:21:00.150 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.keep_log_file_num: 1000 2024-09-10T03:21:00.150 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.recycle_log_file_num: 0 2024-09-10T03:21:00.150 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.allow_fallocate: 1 2024-09-10T03:21:00.150 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.allow_mmap_reads: 0 2024-09-10T03:21:00.150 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.allow_mmap_writes: 0 2024-09-10T03:21:00.150 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.use_direct_reads: 0 2024-09-10T03:21:00.150 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-10T03:21:00.150 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.create_missing_column_families: 0 2024-09-10T03:21:00.150 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.db_log_dir: 2024-09-10T03:21:00.150 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi164/store.db 2024-09-10T03:21:00.151 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-10T03:21:00.151 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-10T03:21:00.151 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-10T03:21:00.151 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-10T03:21:00.151 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-10T03:21:00.151 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-10T03:21:00.152 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.advise_random_on_open: 1 2024-09-10T03:21:00.152 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.db_write_buffer_size: 0 2024-09-10T03:21:00.152 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.write_buffer_manager: 0x55bf6b5d02d0 2024-09-10T03:21:00.152 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-10T03:21:00.152 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-10T03:21:00.152 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-10T03:21:00.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-10T03:21:00.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.rate_limiter: (nil) 2024-09-10T03:21:00.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-10T03:21:00.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.wal_recovery_mode: 2 2024-09-10T03:21:00.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.enable_thread_tracking: 0 2024-09-10T03:21:00.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.enable_pipelined_write: 0 2024-09-10T03:21:00.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.unordered_write: 0 2024-09-10T03:21:00.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-10T03:21:00.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-10T03:21:00.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-10T03:21:00.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-10T03:21:00.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.row_cache: None 2024-09-10T03:21:00.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.wal_filter: None 2024-09-10T03:21:00.153 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-10T03:21:00.154 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.allow_ingest_behind: 0 2024-09-10T03:21:00.154 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.preserve_deletes: 0 2024-09-10T03:21:00.154 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.two_write_queues: 0 2024-09-10T03:21:00.154 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.manual_wal_flush: 0 2024-09-10T03:21:00.154 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.atomic_flush: 0 2024-09-10T03:21:00.154 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-10T03:21:00.154 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-10T03:21:00.154 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-10T03:21:00.154 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.log_readahead_size: 0 2024-09-10T03:21:00.154 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-10T03:21:00.154 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.best_efforts_recovery: 0 2024-09-10T03:21:00.154 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-10T03:21:00.154 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-10T03:21:00.154 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.allow_data_in_errors: 0 2024-09-10T03:21:00.154 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.db_host_id: __hostname__ 2024-09-10T03:21:00.155 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_background_jobs: 2 2024-09-10T03:21:00.155 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_background_compactions: -1 2024-09-10T03:21:00.155 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_subcompactions: 1 2024-09-10T03:21:00.155 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-10T03:21:00.155 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-10T03:21:00.155 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-10T03:21:00.155 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_total_wal_size: 0 2024-09-10T03:21:00.155 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-10T03:21:00.155 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-10T03:21:00.155 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-10T03:21:00.155 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-10T03:21:00.155 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_open_files: -1 2024-09-10T03:21:00.155 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.bytes_per_sync: 0 2024-09-10T03:21:00.155 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-10T03:21:00.156 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-10T03:21:00.156 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compaction_readahead_size: 0 2024-09-10T03:21:00.156 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_background_flushes: -1 2024-09-10T03:21:00.156 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Compression algorithms supported: 2024-09-10T03:21:00.156 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-10T03:21:00.156 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: kZSTD supported: 0 2024-09-10T03:21:00.156 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: kXpressCompression supported: 0 2024-09-10T03:21:00.156 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: kLZ4HCCompression supported: 1 2024-09-10T03:21:00.156 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: kLZ4Compression supported: 1 2024-09-10T03:21:00.156 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: kBZip2Compression supported: 0 2024-09-10T03:21:00.156 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: kZlibCompression supported: 1 2024-09-10T03:21:00.156 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: kSnappyCompression supported: 1 2024-09-10T03:21:00.156 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-10T03:21:00.156 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi164/store.db/MANIFEST-000003 2024-09-10T03:21:00.156 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-10T03:21:00.157 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-10T03:21:00.157 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.merge_operator: 2024-09-10T03:21:00.157 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compaction_filter: None 2024-09-10T03:21:00.157 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compaction_filter_factory: None 2024-09-10T03:21:00.157 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.sst_partitioner_factory: None 2024-09-10T03:21:00.157 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-10T03:21:00.157 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-10T03:21:00.157 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55bf6b4abd00) 2024-09-10T03:21:00.157 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: cache_index_and_filter_blocks: 1 2024-09-10T03:21:00.157 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-10T03:21:00.157 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-10T03:21:00.157 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: pin_top_level_index_and_filter: 1 2024-09-10T03:21:00.157 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: index_type: 0 2024-09-10T03:21:00.157 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: data_block_index_type: 0 2024-09-10T03:21:00.158 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: index_shortening: 1 2024-09-10T03:21:00.158 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-10T03:21:00.158 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: hash_index_allow_collision: 1 2024-09-10T03:21:00.158 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: checksum: 1 2024-09-10T03:21:00.158 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: no_block_cache: 0 2024-09-10T03:21:00.158 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_cache: 0x55bf6b516170 2024-09-10T03:21:00.158 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_cache_name: BinnedLRUCache 2024-09-10T03:21:00.158 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_cache_options: 2024-09-10T03:21:00.158 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: capacity : 536870912 2024-09-10T03:21:00.158 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: num_shard_bits : 4 2024-09-10T03:21:00.158 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: strict_capacity_limit : 0 2024-09-10T03:21:00.158 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: high_pri_pool_ratio: 0.000 2024-09-10T03:21:00.158 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_cache_compressed: (nil) 2024-09-10T03:21:00.158 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: persistent_cache: (nil) 2024-09-10T03:21:00.159 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_size: 4096 2024-09-10T03:21:00.159 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_size_deviation: 10 2024-09-10T03:21:00.159 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_restart_interval: 16 2024-09-10T03:21:00.159 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: index_block_restart_interval: 1 2024-09-10T03:21:00.159 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: metadata_block_size: 4096 2024-09-10T03:21:00.159 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: partition_filters: 0 2024-09-10T03:21:00.159 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: use_delta_encoding: 1 2024-09-10T03:21:00.159 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-10T03:21:00.159 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: whole_key_filtering: 1 2024-09-10T03:21:00.159 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: verify_compression: 0 2024-09-10T03:21:00.159 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: read_amp_bytes_per_bit: 0 2024-09-10T03:21:00.159 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: format_version: 4 2024-09-10T03:21:00.159 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: enable_index_compression: 1 2024-09-10T03:21:00.159 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_align: 0 2024-09-10T03:21:00.160 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.write_buffer_size: 33554432 2024-09-10T03:21:00.160 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_write_buffer_number: 2 2024-09-10T03:21:00.160 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compression: NoCompression 2024-09-10T03:21:00.160 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.bottommost_compression: Disabled 2024-09-10T03:21:00.160 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.prefix_extractor: nullptr 2024-09-10T03:21:00.160 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-10T03:21:00.160 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.num_levels: 7 2024-09-10T03:21:00.160 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-10T03:21:00.160 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-10T03:21:00.160 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-10T03:21:00.160 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-10T03:21:00.160 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-10T03:21:00.160 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-10T03:21:00.160 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-10T03:21:00.161 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-10T03:21:00.161 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-10T03:21:00.161 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-10T03:21:00.161 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-10T03:21:00.161 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compression_opts.level: 32767 2024-09-10T03:21:00.161 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compression_opts.strategy: 0 2024-09-10T03:21:00.161 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-10T03:21:00.161 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-10T03:21:00.161 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-10T03:21:00.161 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compression_opts.enabled: false 2024-09-10T03:21:00.161 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-10T03:21:00.161 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-10T03:21:00.161 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-10T03:21:00.161 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.target_file_size_base: 67108864 2024-09-10T03:21:00.162 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-10T03:21:00.162 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-10T03:21:00.162 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-10T03:21:00.162 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-10T03:21:00.162 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-10T03:21:00.162 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-10T03:21:00.162 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-10T03:21:00.162 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-10T03:21:00.162 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-10T03:21:00.162 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-10T03:21:00.162 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-10T03:21:00.162 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-10T03:21:00.162 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-10T03:21:00.162 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.arena_block_size: 4194304 2024-09-10T03:21:00.163 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-10T03:21:00.163 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-10T03:21:00.163 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-10T03:21:00.163 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.disable_auto_compactions: 0 2024-09-10T03:21:00.163 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-10T03:21:00.163 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-10T03:21:00.163 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-10T03:21:00.163 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-10T03:21:00.163 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-10T03:21:00.163 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-10T03:21:00.163 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-10T03:21:00.163 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-10T03:21:00.163 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-10T03:21:00.164 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-10T03:21:00.164 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.table_properties_collectors: 2024-09-10T03:21:00.164 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.inplace_update_support: 0 2024-09-10T03:21:00.164 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-10T03:21:00.164 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-10T03:21:00.164 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-10T03:21:00.164 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-10T03:21:00.164 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.bloom_locality: 0 2024-09-10T03:21:00.164 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.max_successive_merges: 0 2024-09-10T03:21:00.164 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-10T03:21:00.164 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.paranoid_file_checks: 0 2024-09-10T03:21:00.164 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.force_consistency_checks: 1 2024-09-10T03:21:00.164 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.report_bg_io_stats: 0 2024-09-10T03:21:00.164 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.ttl: 2592000 2024-09-10T03:21:00.165 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-10T03:21:00.165 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.enable_blob_files: false 2024-09-10T03:21:00.165 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.min_blob_size: 0 2024-09-10T03:21:00.165 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.blob_file_size: 268435456 2024-09-10T03:21:00.165 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-10T03:21:00.165 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-10T03:21:00.165 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-10T03:21:00.165 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi164/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-10T03:21:00.165 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-09-10T03:21:00.165 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-09-10T03:21:00.165 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725938460058241, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-10T03:21:00.165 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-09-10T03:21:00.165 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-10T03:21:00.165 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725938460058553, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1725938460, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "636c0bd4-9992-4abe-9479-5b4e9b327fd2", "db_session_id": "M348FH1U3K8D99GIRARQ"}} 2024-09-10T03:21:00.166 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-09-10T03:21:00.166 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725938460058876, "job": 1, "event": "recovery_finished"} 2024-09-10T03:21:00.166 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi164/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-10T03:21:00.166 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55bf6b4fca80 2024-09-10T03:21:00.166 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: DB pointer 0x55bf6b50c000 2024-09-10T03:21:00.166 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-10T03:21:00.166 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-10T03:21:00.166 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ** DB Stats ** 2024-09-10T03:21:00.166 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-10T03:21:00.166 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-10T03:21:00.166 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-10T03:21:00.166 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-10T03:21:00.166 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-10T03:21:00.166 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-10T03:21:00.167 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-10T03:21:00.167 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-09-10T03:21:00.167 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ** Compaction Stats [default] ** 2024-09-10T03:21:00.167 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-10T03:21:00.167 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-10T03:21:00.167 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-10T03:21:00.167 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-10T03:21:00.167 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-10T03:21:00.167 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-09-10T03:21:00.167 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ** Compaction Stats [default] ** 2024-09-10T03:21:00.167 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-10T03:21:00.167 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-10T03:21:00.167 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-10T03:21:00.167 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-10T03:21:00.168 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-10T03:21:00.168 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-10T03:21:00.168 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-10T03:21:00.168 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-10T03:21:00.168 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-10T03:21:00.168 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Cumulative compaction: 0.00 GB write, 0.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-10T03:21:00.168 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Interval compaction: 0.00 GB write, 0.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-10T03:21:00.168 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-10T03:21:00.168 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-09-10T03:21:00.168 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-10T03:21:00.168 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-09-10T03:21:00.168 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ** Compaction Stats [default] ** 2024-09-10T03:21:00.168 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-10T03:21:00.168 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-10T03:21:00.169 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-10T03:21:00.169 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-10T03:21:00.169 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-10T03:21:00.169 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-09-10T03:21:00.169 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ** Compaction Stats [default] ** 2024-09-10T03:21:00.169 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-10T03:21:00.169 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-10T03:21:00.169 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-10T03:21:00.169 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-10T03:21:00.169 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-10T03:21:00.169 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-10T03:21:00.169 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-10T03:21:00.169 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-10T03:21:00.169 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-10T03:21:00.170 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Cumulative compaction: 0.00 GB write, 0.53 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-10T03:21:00.170 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-10T03:21:00.170 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-10T03:21:00.170 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-09-10T03:21:00.170 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-10T03:21:00.170 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mon.smithi164 does not exist in monmap, will attempt to join an existing cluster 2024-09-10T03:21:00.170 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: using public_addr v2:172.21.15.164:0/0 -> [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] 2024-09-10T03:21:00.171 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: starting mon.smithi164 rank -1 at public addrs [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] at bind addrs [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi164 fsid 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:21:00.171 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mon.smithi164@-1(???) e0 preinit fsid 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:21:00.171 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mon.smithi164@-1(synchronizing).mds e1 new map 2024-09-10T03:21:00.171 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mon.smithi164@-1(synchronizing).mds e1 print_map 2024-09-10T03:21:00.171 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: e1 2024-09-10T03:21:00.171 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-10T03:21:00.171 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-10T03:21:00.171 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: legacy client fscid: -1 2024-09-10T03:21:00.171 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-09-10T03:21:00.172 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: No filesystems configured 2024-09-10T03:21:00.172 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mon.smithi164@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-10T03:21:00.172 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mon.smithi164@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-10T03:21:00.172 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mon.smithi164@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-10T03:21:00.172 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mon.smithi164@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-10T03:21:00.172 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mon.smithi164@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-10T03:21:00.172 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mon.smithi164@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-10T03:21:00.172 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mon.smithi164@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-09-10T03:21:00.172 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mon.smithi164@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-10T03:21:00.172 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mon.smithi164@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-10T03:21:00.172 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mon.smithi164@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-10T03:21:00.172 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mon.smithi164@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-10T03:21:00.172 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: Added host smithi164 2024-09-10T03:21:00.173 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:00.173 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.173 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi130", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-10T03:21:00.173 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi130", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-10T03:21:00.173 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:00.173 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: Deploying daemon crash.smithi130 on smithi130 2024-09-10T03:21:00.173 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:00.173 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.173 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:00.173 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-10T03:21:00.173 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.173 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.173 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.173 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-10T03:21:00.174 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-10T03:21:00.174 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.174 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: Deploying daemon grafana.smithi130 on smithi130 2024-09-10T03:21:00.174 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:00.174 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/349461942' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-10T03:21:00.174 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/349461942' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-10T03:21:00.174 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: osdmap e4: 0 total, 0 up, 0 in 2024-09-10T03:21:00.174 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.174 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:00.174 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:00.174 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi130:172.21.15.130=smithi130;smithi164:172.21.15.164=smithi164", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:21:00.174 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: Saving service mon spec with placement smithi130:172.21.15.130=smithi130;smithi164:172.21.15.164=smithi164;count:2 2024-09-10T03:21:00.174 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.174 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:00.174 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='client.? 172.21.15.164:0/885199302' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:21:00.175 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:00.175 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='client.? 172.21.15.164:0/923517571' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:21:00.175 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:00.175 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.175 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: Deploying daemon node-exporter.smithi130 on smithi130 2024-09-10T03:21:00.175 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.175 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:00.175 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='client.? 172.21.15.164:0/2442967967' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:21:00.175 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.175 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-10T03:21:00.175 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:00.175 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14160 172.21.15.130:0/3020118834' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-10T03:21:00.175 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mgrmap e14: smithi130.axcmel(active, since 43s) 2024-09-10T03:21:00.175 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='client.? 172.21.15.164:0/3102429567' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:21:00.176 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='client.? 172.21.15.164:0/33711504' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:21:00.176 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: Active manager daemon smithi130.axcmel restarted 2024-09-10T03:21:00.176 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: Activating manager daemon smithi130.axcmel 2024-09-10T03:21:00.176 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: osdmap e5: 0 total, 0 up, 0 in 2024-09-10T03:21:00.176 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mgrmap e15: smithi130.axcmel(active, starting, since 0.183151s) 2024-09-10T03:21:00.176 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi130"}]: dispatch 2024-09-10T03:21:00.176 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr metadata", "who": "smithi130.axcmel", "id": "smithi130.axcmel"}]: dispatch 2024-09-10T03:21:00.176 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-10T03:21:00.176 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-10T03:21:00.176 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-10T03:21:00.176 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: Manager daemon smithi130.axcmel is now available 2024-09-10T03:21:00.176 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:21:00.176 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:21:00.176 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:00.177 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:21:00.177 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.177 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:21:00.177 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:21:00.177 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: [10/Sep/2024:03:20:41] ENGINE Bus STARTING 2024-09-10T03:21:00.177 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: [10/Sep/2024:03:20:41] ENGINE Serving on https://172.21.15.130:7150 2024-09-10T03:21:00.177 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: [10/Sep/2024:03:20:41] ENGINE Bus STARTED 2024-09-10T03:21:00.177 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.177 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mgrmap e16: smithi130.axcmel(active, since 1.18589s) 2024-09-10T03:21:00.177 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.177 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.177 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='client.? 172.21.15.164:0/4204399153' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:21:00.177 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mgrmap e17: smithi130.axcmel(active, since 2s) 2024-09-10T03:21:00.177 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.177 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.178 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.178 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd/host:smithi130", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:21:00.178 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.178 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: Updating smithi130:/etc/ceph/ceph.conf 2024-09-10T03:21:00.178 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-09-10T03:21:00.178 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.178 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.178 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd/host:smithi164", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:21:00.178 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.178 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi164", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-10T03:21:00.178 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi164", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-10T03:21:00.178 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:00.178 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: Deploying daemon crash.smithi164 on smithi164 2024-09-10T03:21:00.178 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='client.? 172.21.15.164:0/3411075316' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:21:00.179 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='client.? 172.21.15.164:0/738441756' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:21:00.179 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.179 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.bkfwap", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-10T03:21:00.179 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.bkfwap", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-10T03:21:00.179 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-10T03:21:00.179 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:00.179 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: Deploying daemon mgr.smithi164.bkfwap on smithi164 2024-09-10T03:21:00.179 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='client.? 172.21.15.164:0/357722886' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:21:00.179 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.179 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-10T03:21:00.179 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:00.179 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: Deploying daemon mon.smithi164 on smithi164 2024-09-10T03:21:00.179 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:00.179 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mon.smithi164@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-09-10T03:21:00.180 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 ceph-mon[25711]: mon.smithi164@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-09-10T03:21:00.180 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 bash[25629]: 7a928b0b045c1c66777736a5fe091bdfe77eb7ab7e9b47ec99384fc8460e6318 2024-09-10T03:21:00.180 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:00 smithi164 systemd[1]: Started Ceph mon.smithi164 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:21:01.849 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-10T03:21:01.849 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph mon dump -f json 2024-09-10T03:21:05.390 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi130"}]: dispatch 2024-09-10T03:21:05.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: mon.smithi130 calling monitor election 2024-09-10T03:21:05.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:21:05.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:21:05.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:05.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/3281948765' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/crt"}]: dispatch 2024-09-10T03:21:05.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:21:05.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: mon.smithi164 calling monitor election 2024-09-10T03:21:05.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:21:05.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:05.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:21:05.392 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:21:05.392 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: mon.smithi130 is new leader, mons smithi130,smithi164 in quorum (ranks 0,1) 2024-09-10T03:21:05.392 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: monmap e2: 2 mons at {smithi130=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],smithi164=[v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0]} 2024-09-10T03:21:05.392 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: fsmap 2024-09-10T03:21:05.392 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: osdmap e5: 0 total, 0 up, 0 in 2024-09-10T03:21:05.392 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: mgrmap e17: smithi130.axcmel(active, since 23s) 2024-09-10T03:21:05.392 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: Standby manager daemon smithi164.bkfwap started 2024-09-10T03:21:05.392 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/3281948765' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-10T03:21:05.392 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: overall HEALTH_OK 2024-09-10T03:21:05.392 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/3281948765' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/key"}]: dispatch 2024-09-10T03:21:05.392 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/3281948765' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-10T03:21:05.392 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:05 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:05.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi130"}]: dispatch 2024-09-10T03:21:05.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: mon.smithi130 calling monitor election 2024-09-10T03:21:05.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:21:05.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:21:05.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:05.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.164:0/3281948765' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/crt"}]: dispatch 2024-09-10T03:21:05.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:21:05.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: mon.smithi164 calling monitor election 2024-09-10T03:21:05.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:21:05.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:05.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:21:05.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:21:05.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: mon.smithi130 is new leader, mons smithi130,smithi164 in quorum (ranks 0,1) 2024-09-10T03:21:05.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: monmap e2: 2 mons at {smithi130=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],smithi164=[v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0]} 2024-09-10T03:21:05.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: fsmap 2024-09-10T03:21:05.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: osdmap e5: 0 total, 0 up, 0 in 2024-09-10T03:21:05.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: mgrmap e17: smithi130.axcmel(active, since 23s) 2024-09-10T03:21:05.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: Standby manager daemon smithi164.bkfwap started 2024-09-10T03:21:05.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.164:0/3281948765' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-10T03:21:05.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: overall HEALTH_OK 2024-09-10T03:21:05.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.164:0/3281948765' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/key"}]: dispatch 2024-09-10T03:21:05.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.164:0/3281948765' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-10T03:21:05.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:05 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:06.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:06 smithi164 ceph-mon[25711]: Deploying daemon node-exporter.smithi164 on smithi164 2024-09-10T03:21:06.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:06 smithi164 ceph-mon[25711]: mgrmap e18: smithi130.axcmel(active, since 23s), standbys: smithi164.bkfwap 2024-09-10T03:21:06.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:06 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr metadata", "who": "smithi164.bkfwap", "id": "smithi164.bkfwap"}]: dispatch 2024-09-10T03:21:06.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:06 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:21:06.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:06 smithi130 ceph-mon[23614]: Deploying daemon node-exporter.smithi164 on smithi164 2024-09-10T03:21:06.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:06 smithi130 ceph-mon[23614]: mgrmap e18: smithi130.axcmel(active, since 23s), standbys: smithi164.bkfwap 2024-09-10T03:21:06.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:06 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr metadata", "who": "smithi164.bkfwap", "id": "smithi164.bkfwap"}]: dispatch 2024-09-10T03:21:06.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:06 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:21:06.714 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:21:06.714 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":2,"fsid":"5829498e-6f23-11ef-bcea-c7b262605968","modified":"2024-09-10T03:21:00.086564Z","created":"2024-09-10T03:18:46.462733Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi130","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi164","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:3300","nonce":0},{"type":"v1","addr":"172.21.15.164:6789","nonce":0}]},"addr":"172.21.15.164:6789/0","public_addr":"172.21.15.164:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-10T03:21:06.715 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 2 2024-09-10T03:21:07.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:07 smithi164 ceph-mon[25711]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:07.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:07 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:07.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:07 smithi164 ceph-mon[25711]: from='client.? 172.21.15.164:0/304676433' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:21:07.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:07 smithi130 ceph-mon[23614]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:07.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:07 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:07.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:07 smithi130 ceph-mon[23614]: from='client.? 172.21.15.164:0/304676433' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-10T03:21:08.081 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-10T03:21:08.081 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph config generate-minimal-conf 2024-09-10T03:21:09.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:09 smithi130 ceph-mon[23614]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:09.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:09 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:09.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:09 smithi164 ceph-mon[25711]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:09.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:09 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:09.998 INFO:teuthology.orchestra.run.smithi130.stdout:# minimal ceph.conf for 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:21:09.998 INFO:teuthology.orchestra.run.smithi130.stdout:[global] 2024-09-10T03:21:09.998 INFO:teuthology.orchestra.run.smithi130.stdout: fsid = 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:21:09.998 INFO:teuthology.orchestra.run.smithi130.stdout: mon_host = [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] 2024-09-10T03:21:10.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:10 smithi130 ceph-mon[23614]: Deploying daemon prometheus.smithi130 on smithi130 2024-09-10T03:21:10.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:10 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3940551654' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:10.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:10 smithi164 ceph-mon[25711]: Deploying daemon prometheus.smithi130 on smithi130 2024-09-10T03:21:10.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:10 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/3940551654' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:10.615 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-10T03:21:10.615 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-09-10T03:21:10.615 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/ceph/ceph.conf 2024-09-10T03:21:10.643 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-09-10T03:21:10.643 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-10T03:21:10.708 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-10T03:21:10.708 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/etc/ceph/ceph.conf 2024-09-10T03:21:10.742 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-10T03:21:10.742 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-10T03:21:10.811 DEBUG:tasks.cephadm:set 0 configs 2024-09-10T03:21:10.811 INFO:tasks.cephadm:Deploying OSDs... 2024-09-10T03:21:10.811 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-09-10T03:21:10.812 DEBUG:teuthology.orchestra.run.smithi130:> dd if=/scratch_devs of=/dev/stdout 2024-09-10T03:21:10.828 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-10T03:21:10.828 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_1 2024-09-10T03:21:10.884 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-10T03:21:10.884 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-10T03:21:10.884 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-09-10T03:21:10.885 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-10T03:21:10.885 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2024-09-10T03:21:10.885 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2024-09-10 03:20:31.960325248 +0000 2024-09-10T03:21:10.885 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2024-09-10 03:18:05.551746159 +0000 2024-09-10T03:21:10.885 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2024-09-10 03:18:05.551746159 +0000 2024-09-10T03:21:10.885 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: 2024-09-10 03:18:05.551746159 +0000 2024-09-10T03:21:10.885 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-10T03:21:10.948 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2024-09-10T03:21:10.948 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2024-09-10T03:21:10.948 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000196745 s, 2.6 MB/s 2024-09-10T03:21:10.949 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-10T03:21:11.007 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_2 2024-09-10T03:21:11.066 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-10T03:21:11.066 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-10T03:21:11.066 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-09-10T03:21:11.066 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-10T03:21:11.066 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2024-09-10T03:21:11.066 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2024-09-10 03:20:31.961325230 +0000 2024-09-10T03:21:11.066 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2024-09-10 03:18:05.882740099 +0000 2024-09-10T03:21:11.066 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2024-09-10 03:18:05.882740099 +0000 2024-09-10T03:21:11.066 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: 2024-09-10 03:18:05.882740099 +0000 2024-09-10T03:21:11.067 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-10T03:21:11.136 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2024-09-10T03:21:11.137 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2024-09-10T03:21:11.137 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000194613 s, 2.6 MB/s 2024-09-10T03:21:11.138 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-10T03:21:11.198 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_3 2024-09-10T03:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-10T03:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-10T03:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-09-10T03:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-10T03:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2024-09-10T03:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2024-09-10 03:20:31.961325230 +0000 2024-09-10T03:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2024-09-10 03:18:06.224733837 +0000 2024-09-10T03:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2024-09-10 03:18:06.224733837 +0000 2024-09-10T03:21:11.256 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: 2024-09-10 03:18:06.224733837 +0000 2024-09-10T03:21:11.256 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-10T03:21:11.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:11 smithi130 ceph-mon[23614]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:11.289 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2024-09-10T03:21:11.289 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2024-09-10T03:21:11.289 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000189824 s, 2.7 MB/s 2024-09-10T03:21:11.291 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-10T03:21:11.349 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_4 2024-09-10T03:21:11.407 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-10T03:21:11.407 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-10T03:21:11.407 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-09-10T03:21:11.407 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-10T03:21:11.407 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2024-09-10T03:21:11.407 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2024-09-10 03:20:31.961325230 +0000 2024-09-10T03:21:11.408 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2024-09-10 03:18:06.547727923 +0000 2024-09-10T03:21:11.408 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2024-09-10 03:18:06.547727923 +0000 2024-09-10T03:21:11.408 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: 2024-09-10 03:18:06.547727923 +0000 2024-09-10T03:21:11.408 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-10T03:21:11.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:11 smithi164 ceph-mon[25711]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:11.470 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2024-09-10T03:21:11.471 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2024-09-10T03:21:11.471 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000181831 s, 2.8 MB/s 2024-09-10T03:21:11.472 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-10T03:21:11.529 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2024-09-10T03:21:11.529 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/scratch_devs of=/dev/stdout 2024-09-10T03:21:11.548 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-10T03:21:11.549 DEBUG:teuthology.orchestra.run.smithi164:> stat /dev/vg_nvme/lv_1 2024-09-10T03:21:11.607 INFO:teuthology.orchestra.run.smithi164.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-10T03:21:11.607 INFO:teuthology.orchestra.run.smithi164.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-10T03:21:11.607 INFO:teuthology.orchestra.run.smithi164.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-09-10T03:21:11.607 INFO:teuthology.orchestra.run.smithi164.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-10T03:21:11.607 INFO:teuthology.orchestra.run.smithi164.stdout:Context: system_u:object_r:device_t:s0 2024-09-10T03:21:11.607 INFO:teuthology.orchestra.run.smithi164.stdout:Access: 2024-09-10 03:21:05.680005407 +0000 2024-09-10T03:21:11.607 INFO:teuthology.orchestra.run.smithi164.stdout:Modify: 2024-09-10 03:18:05.557114235 +0000 2024-09-10T03:21:11.607 INFO:teuthology.orchestra.run.smithi164.stdout:Change: 2024-09-10 03:18:05.557114235 +0000 2024-09-10T03:21:11.608 INFO:teuthology.orchestra.run.smithi164.stdout: Birth: 2024-09-10 03:18:05.557114235 +0000 2024-09-10T03:21:11.608 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-10T03:21:11.675 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records in 2024-09-10T03:21:11.675 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records out 2024-09-10T03:21:11.675 INFO:teuthology.orchestra.run.smithi164.stderr:512 bytes copied, 0.000278825 s, 1.8 MB/s 2024-09-10T03:21:11.676 DEBUG:teuthology.orchestra.run.smithi164:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-10T03:21:11.735 DEBUG:teuthology.orchestra.run.smithi164:> stat /dev/vg_nvme/lv_2 2024-09-10T03:21:11.792 INFO:teuthology.orchestra.run.smithi164.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-10T03:21:11.793 INFO:teuthology.orchestra.run.smithi164.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-10T03:21:11.793 INFO:teuthology.orchestra.run.smithi164.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-09-10T03:21:11.793 INFO:teuthology.orchestra.run.smithi164.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-10T03:21:11.793 INFO:teuthology.orchestra.run.smithi164.stdout:Context: system_u:object_r:device_t:s0 2024-09-10T03:21:11.793 INFO:teuthology.orchestra.run.smithi164.stdout:Access: 2024-09-10 03:21:05.680005407 +0000 2024-09-10T03:21:11.793 INFO:teuthology.orchestra.run.smithi164.stdout:Modify: 2024-09-10 03:18:05.899108496 +0000 2024-09-10T03:21:11.793 INFO:teuthology.orchestra.run.smithi164.stdout:Change: 2024-09-10 03:18:05.899108496 +0000 2024-09-10T03:21:11.793 INFO:teuthology.orchestra.run.smithi164.stdout: Birth: 2024-09-10 03:18:05.899108496 +0000 2024-09-10T03:21:11.793 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-10T03:21:11.858 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records in 2024-09-10T03:21:11.859 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records out 2024-09-10T03:21:11.859 INFO:teuthology.orchestra.run.smithi164.stderr:512 bytes copied, 0.000244539 s, 2.1 MB/s 2024-09-10T03:21:11.861 DEBUG:teuthology.orchestra.run.smithi164:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-10T03:21:11.918 DEBUG:teuthology.orchestra.run.smithi164:> stat /dev/vg_nvme/lv_3 2024-09-10T03:21:11.974 INFO:teuthology.orchestra.run.smithi164.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-10T03:21:11.975 INFO:teuthology.orchestra.run.smithi164.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-10T03:21:11.975 INFO:teuthology.orchestra.run.smithi164.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-09-10T03:21:11.975 INFO:teuthology.orchestra.run.smithi164.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-10T03:21:11.975 INFO:teuthology.orchestra.run.smithi164.stdout:Context: system_u:object_r:device_t:s0 2024-09-10T03:21:11.975 INFO:teuthology.orchestra.run.smithi164.stdout:Access: 2024-09-10 03:21:05.680005407 +0000 2024-09-10T03:21:11.975 INFO:teuthology.orchestra.run.smithi164.stdout:Modify: 2024-09-10 03:18:06.216103177 +0000 2024-09-10T03:21:11.975 INFO:teuthology.orchestra.run.smithi164.stdout:Change: 2024-09-10 03:18:06.216103177 +0000 2024-09-10T03:21:11.975 INFO:teuthology.orchestra.run.smithi164.stdout: Birth: 2024-09-10 03:18:06.216103177 +0000 2024-09-10T03:21:11.975 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-10T03:21:12.040 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records in 2024-09-10T03:21:12.040 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records out 2024-09-10T03:21:12.040 INFO:teuthology.orchestra.run.smithi164.stderr:512 bytes copied, 0.000199627 s, 2.6 MB/s 2024-09-10T03:21:12.041 DEBUG:teuthology.orchestra.run.smithi164:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-10T03:21:12.098 DEBUG:teuthology.orchestra.run.smithi164:> stat /dev/vg_nvme/lv_4 2024-09-10T03:21:12.154 INFO:teuthology.orchestra.run.smithi164.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-10T03:21:12.154 INFO:teuthology.orchestra.run.smithi164.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-10T03:21:12.154 INFO:teuthology.orchestra.run.smithi164.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-10T03:21:12.154 INFO:teuthology.orchestra.run.smithi164.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-10T03:21:12.154 INFO:teuthology.orchestra.run.smithi164.stdout:Context: system_u:object_r:device_t:s0 2024-09-10T03:21:12.154 INFO:teuthology.orchestra.run.smithi164.stdout:Access: 2024-09-10 03:21:05.681005390 +0000 2024-09-10T03:21:12.155 INFO:teuthology.orchestra.run.smithi164.stdout:Modify: 2024-09-10 03:18:06.560097405 +0000 2024-09-10T03:21:12.155 INFO:teuthology.orchestra.run.smithi164.stdout:Change: 2024-09-10 03:18:06.560097405 +0000 2024-09-10T03:21:12.155 INFO:teuthology.orchestra.run.smithi164.stdout: Birth: 2024-09-10 03:18:06.560097405 +0000 2024-09-10T03:21:12.155 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-10T03:21:12.218 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records in 2024-09-10T03:21:12.219 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records out 2024-09-10T03:21:12.219 INFO:teuthology.orchestra.run.smithi164.stderr:512 bytes copied, 0.000216489 s, 2.4 MB/s 2024-09-10T03:21:12.220 DEBUG:teuthology.orchestra.run.smithi164:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-10T03:21:12.277 INFO:tasks.cephadm:Deploying osd.0 on smithi130 with /dev/vg_nvme/lv_4... 2024-09-10T03:21:12.277 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-10T03:21:12.940 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:12 smithi130 ceph-mon[23614]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:12.940 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:12 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:12.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:12 smithi164 ceph-mon[25711]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:12.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:12 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:14.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:14 smithi164 ceph-mon[25711]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:15.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:14 smithi130 ceph-mon[23614]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:16.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:16 smithi164 ceph-mon[25711]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:17.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:16 smithi130 ceph-mon[23614]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:18.810 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:18 smithi130 ceph-mon[23614]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:18.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:18 smithi164 ceph-mon[25711]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:20.912 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:20 smithi130 ceph-mon[23614]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:20.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:20 smithi164 ceph-mon[25711]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:22.508 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:22 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:22.509 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:22 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:21:22.509 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:22 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:22.509 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:22 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:21:22.509 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:22 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:22.596 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:22 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:22.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:22 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:21:22.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:22 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:22.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:22 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:21:22.597 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:22 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:22.933 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:21:22.952 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph orch daemon add osd smithi130:vg_nvme/lv_4 2024-09-10T03:21:23.591 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:23 smithi130 ceph-mon[23614]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:23.591 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:23 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:23.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:23 smithi164 ceph-mon[25711]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:23.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:23 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:24.492 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:24 smithi130 ceph-mon[23614]: Updating smithi164:/etc/ceph/ceph.conf 2024-09-10T03:21:24.493 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:24 smithi130 ceph-mon[23614]: Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2024-09-10T03:21:24.493 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:24 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:24.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:24 smithi164 ceph-mon[25711]: Updating smithi164:/etc/ceph/ceph.conf 2024-09-10T03:21:24.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:24 smithi164 ceph-mon[25711]: Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2024-09-10T03:21:24.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:24 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:25.591 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:25 smithi130 ceph-mon[23614]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:25.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:25 smithi164 ceph-mon[25711]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:26.416 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:26 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-10T03:21:26.417 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:26 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-10T03:21:26.417 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:26 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:26.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:26 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-10T03:21:26.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:26 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-10T03:21:26.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:26 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:27.683 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:27 smithi130 ceph-mon[23614]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:27.683 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:27 smithi130 ceph-mon[23614]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:21:27.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:27 smithi164 ceph-mon[25711]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:27.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:27 smithi164 ceph-mon[25711]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:21:28.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:28 smithi130 ceph-mon[23614]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:28.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:28 smithi164 ceph-mon[25711]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:31.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:31 smithi130 ceph-mon[23614]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:31.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:31 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:31.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:31 smithi130 ceph-mon[23614]: Updating smithi130:/etc/ceph/ceph.conf 2024-09-10T03:21:31.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:31 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:31.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:31 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:31.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:31 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-10T03:21:31.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:31 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-10T03:21:31.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:31 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:31.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:31 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3900239177' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "28358d54-c6fe-475e-b7ac-60c43d222eae"}]: dispatch 2024-09-10T03:21:31.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:31 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3900239177' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "28358d54-c6fe-475e-b7ac-60c43d222eae"}]': finished 2024-09-10T03:21:31.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:31 smithi130 ceph-mon[23614]: osdmap e6: 1 total, 0 up, 1 in 2024-09-10T03:21:31.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:31 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-10T03:21:31.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:31 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/1397242789' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-10T03:21:31.589 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:31 smithi164 ceph-mon[25711]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:31.590 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:31 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:31.590 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:31 smithi164 ceph-mon[25711]: Updating smithi130:/etc/ceph/ceph.conf 2024-09-10T03:21:31.590 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:31 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:31.590 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:31 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:31.590 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:31 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-10T03:21:31.590 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:31 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-10T03:21:31.590 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:31 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:31.590 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:31 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/3900239177' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "28358d54-c6fe-475e-b7ac-60c43d222eae"}]: dispatch 2024-09-10T03:21:31.590 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:31 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/3900239177' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "28358d54-c6fe-475e-b7ac-60c43d222eae"}]': finished 2024-09-10T03:21:31.590 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:31 smithi164 ceph-mon[25711]: osdmap e6: 1 total, 0 up, 1 in 2024-09-10T03:21:31.590 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:31 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-10T03:21:31.590 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:31 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/1397242789' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-10T03:21:32.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:32 smithi130 ceph-mon[23614]: Reconfiguring mon.smithi130 (unknown last config time)... 2024-09-10T03:21:32.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:32 smithi130 ceph-mon[23614]: Reconfiguring daemon mon.smithi130 on smithi130 2024-09-10T03:21:32.600 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:32 smithi164 ceph-mon[25711]: Reconfiguring mon.smithi130 (unknown last config time)... 2024-09-10T03:21:32.600 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:32 smithi164 ceph-mon[25711]: Reconfiguring daemon mon.smithi130 on smithi130 2024-09-10T03:21:33.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:33 smithi130 ceph-mon[23614]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:33.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:33 smithi164 ceph-mon[25711]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:35.404 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:35 smithi130 ceph-mon[23614]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:35.404 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:35.404 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi130.axcmel", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-10T03:21:35.404 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-10T03:21:35.404 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:35.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:35 smithi164 ceph-mon[25711]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:35.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:35.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi130.axcmel", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-10T03:21:35.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-10T03:21:35.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:36.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:36 smithi130 ceph-mon[23614]: Reconfiguring mgr.smithi130.axcmel (unknown last config time)... 2024-09-10T03:21:36.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:36 smithi130 ceph-mon[23614]: Reconfiguring daemon mgr.smithi130.axcmel on smithi130 2024-09-10T03:21:36.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:36 smithi164 ceph-mon[25711]: Reconfiguring mgr.smithi130.axcmel (unknown last config time)... 2024-09-10T03:21:36.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:36 smithi164 ceph-mon[25711]: Reconfiguring daemon mgr.smithi130.axcmel on smithi130 2024-09-10T03:21:37.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:37 smithi130 ceph-mon[23614]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:37.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:37 smithi164 ceph-mon[25711]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:38.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:38 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:38.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:38 smithi164 ceph-mon[25711]: Reconfiguring alertmanager.smithi130 (dependencies changed)... 2024-09-10T03:21:38.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:38 smithi164 ceph-mon[25711]: Reconfiguring daemon alertmanager.smithi130 on smithi130 2024-09-10T03:21:38.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:38 smithi164 ceph-mon[25711]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:38.725 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:38 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:38.725 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:38 smithi130 ceph-mon[23614]: Reconfiguring alertmanager.smithi130 (dependencies changed)... 2024-09-10T03:21:38.725 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:38 smithi130 ceph-mon[23614]: Reconfiguring daemon alertmanager.smithi130 on smithi130 2024-09-10T03:21:38.725 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:38 smithi130 ceph-mon[23614]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:39.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:39 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-10T03:21:39.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:39 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:39.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:39 smithi164 ceph-mon[25711]: Deploying daemon osd.0 on smithi130 2024-09-10T03:21:39.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:39 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-10T03:21:39.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:39 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:39.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:39 smithi130 ceph-mon[23614]: Deploying daemon osd.0 on smithi130 2024-09-10T03:21:40.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:40 smithi130 ceph-mon[23614]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:40.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:40 smithi164 ceph-mon[25711]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:42.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:42 smithi130 ceph-mon[23614]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:42.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:21:42.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:21:42.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:42 smithi164 ceph-mon[25711]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:42.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:21:42.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:21:44.504 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:44 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:44.504 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:44 smithi130 ceph-mon[23614]: Reconfiguring crash.smithi130 (monmap changed)... 2024-09-10T03:21:44.504 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:44 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi130", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-10T03:21:44.504 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:44 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:44.504 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:44 smithi130 ceph-mon[23614]: Reconfiguring daemon crash.smithi130 on smithi130 2024-09-10T03:21:44.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:44 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:44.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:44 smithi164 ceph-mon[25711]: Reconfiguring crash.smithi130 (monmap changed)... 2024-09-10T03:21:44.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:44 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi130", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-10T03:21:44.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:44 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:44.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:44 smithi164 ceph-mon[25711]: Reconfiguring daemon crash.smithi130 on smithi130 2024-09-10T03:21:45.552 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:45 smithi130 ceph-mon[23614]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:45.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:45 smithi164 ceph-mon[25711]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:47.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:47 smithi130 ceph-mon[23614]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:47.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:47 smithi164 ceph-mon[25711]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:49.706 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:49 smithi130 ceph-mon[23614]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:49.706 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:49 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:49.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:49 smithi164 ceph-mon[25711]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:49.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:49 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:50.871 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:50 smithi130 ceph-mon[23614]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:50.871 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:50 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:50.871 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:50 smithi130 ceph-mon[23614]: Reconfiguring grafana.smithi130 (dependencies changed)... 2024-09-10T03:21:50.871 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:50 smithi130 ceph-mon[23614]: Reconfiguring daemon grafana.smithi130 on smithi130 2024-09-10T03:21:51.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:50 smithi164 ceph-mon[25711]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:51.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:50 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:51.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:50 smithi164 ceph-mon[25711]: Reconfiguring grafana.smithi130 (dependencies changed)... 2024-09-10T03:21:51.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:50 smithi164 ceph-mon[25711]: Reconfiguring daemon grafana.smithi130 on smithi130 2024-09-10T03:21:51.895 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:51 smithi130 ceph-mon[23614]: from='osd.0 [v2:172.21.15.130:6802/1145235369,v1:172.21.15.130:6803/1145235369]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-10T03:21:51.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:51 smithi164 ceph-mon[25711]: from='osd.0 [v2:172.21.15.130:6802/1145235369,v1:172.21.15.130:6803/1145235369]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-10T03:21:52.993 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:52 smithi130 ceph-mon[23614]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:52.993 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:52 smithi130 ceph-mon[23614]: from='osd.0 [v2:172.21.15.130:6802/1145235369,v1:172.21.15.130:6803/1145235369]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-10T03:21:52.993 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:52 smithi130 ceph-mon[23614]: osdmap e7: 1 total, 0 up, 1 in 2024-09-10T03:21:52.993 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:52 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-10T03:21:52.993 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:52 smithi130 ceph-mon[23614]: from='osd.0 [v2:172.21.15.130:6802/1145235369,v1:172.21.15.130:6803/1145235369]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-09-10T03:21:53.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:52 smithi164 ceph-mon[25711]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:53.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:52 smithi164 ceph-mon[25711]: from='osd.0 [v2:172.21.15.130:6802/1145235369,v1:172.21.15.130:6803/1145235369]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-10T03:21:53.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:52 smithi164 ceph-mon[25711]: osdmap e7: 1 total, 0 up, 1 in 2024-09-10T03:21:53.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:52 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-10T03:21:53.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:52 smithi164 ceph-mon[25711]: from='osd.0 [v2:172.21.15.130:6802/1145235369,v1:172.21.15.130:6803/1145235369]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-09-10T03:21:54.073 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:53 smithi130 ceph-mon[23614]: from='osd.0 [v2:172.21.15.130:6802/1145235369,v1:172.21.15.130:6803/1145235369]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-09-10T03:21:54.073 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:53 smithi130 ceph-mon[23614]: osdmap e8: 1 total, 0 up, 1 in 2024-09-10T03:21:54.073 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:53 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-10T03:21:54.073 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:53 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-10T03:21:54.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:53 smithi164 ceph-mon[25711]: from='osd.0 [v2:172.21.15.130:6802/1145235369,v1:172.21.15.130:6803/1145235369]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-09-10T03:21:54.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:53 smithi164 ceph-mon[25711]: osdmap e8: 1 total, 0 up, 1 in 2024-09-10T03:21:54.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:53 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-10T03:21:54.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:53 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-10T03:21:54.872 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:54 smithi164 ceph-mon[25711]: purged_snaps scrub starts 2024-09-10T03:21:54.872 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:54 smithi164 ceph-mon[25711]: purged_snaps scrub ok 2024-09-10T03:21:54.872 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:54 smithi164 ceph-mon[25711]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:54.872 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:54 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-10T03:21:54.872 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:54 smithi164 ceph-mon[25711]: osd.0 [v2:172.21.15.130:6802/1145235369,v1:172.21.15.130:6803/1145235369] boot 2024-09-10T03:21:54.872 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:54 smithi164 ceph-mon[25711]: osdmap e9: 1 total, 1 up, 1 in 2024-09-10T03:21:54.872 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:54 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-10T03:21:54.872 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:54 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:54.872 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:54 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi164", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-10T03:21:54.873 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:54 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:54.990 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:54 smithi130 ceph-mon[23614]: purged_snaps scrub starts 2024-09-10T03:21:54.990 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:54 smithi130 ceph-mon[23614]: purged_snaps scrub ok 2024-09-10T03:21:54.990 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:54 smithi130 ceph-mon[23614]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-10T03:21:54.991 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:54 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-10T03:21:54.991 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:54 smithi130 ceph-mon[23614]: osd.0 [v2:172.21.15.130:6802/1145235369,v1:172.21.15.130:6803/1145235369] boot 2024-09-10T03:21:54.991 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:54 smithi130 ceph-mon[23614]: osdmap e9: 1 total, 1 up, 1 in 2024-09-10T03:21:54.991 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:54 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-10T03:21:54.991 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:54 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:54.991 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:54 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi164", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-10T03:21:54.991 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:54 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:55.983 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 0 on host 'smithi130' 2024-09-10T03:21:56.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:55 smithi130 ceph-mon[23614]: Reconfiguring crash.smithi164 (monmap changed)... 2024-09-10T03:21:56.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:55 smithi130 ceph-mon[23614]: Reconfiguring daemon crash.smithi164 on smithi164 2024-09-10T03:21:56.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:55 smithi130 ceph-mon[23614]: osdmap e10: 1 total, 1 up, 1 in 2024-09-10T03:21:56.203 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:55 smithi164 ceph-mon[25711]: Reconfiguring crash.smithi164 (monmap changed)... 2024-09-10T03:21:56.203 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:55 smithi164 ceph-mon[25711]: Reconfiguring daemon crash.smithi164 on smithi164 2024-09-10T03:21:56.203 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:55 smithi164 ceph-mon[25711]: osdmap e10: 1 total, 1 up, 1 in 2024-09-10T03:21:56.564 DEBUG:teuthology.orchestra.run.smithi130:osd.0> sudo journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.0.service 2024-09-10T03:21:56.567 INFO:tasks.cephadm:Deploying osd.1 on smithi130 with /dev/vg_nvme/lv_3... 2024-09-10T03:21:56.568 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-10T03:21:57.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:56 smithi164 ceph-mon[25711]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:21:57.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:57.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:56 smithi164 ceph-mon[25711]: Reconfiguring mgr.smithi164.bkfwap (monmap changed)... 2024-09-10T03:21:57.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.bkfwap", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-10T03:21:57.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-10T03:21:57.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:57.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:56 smithi164 ceph-mon[25711]: Reconfiguring daemon mgr.smithi164.bkfwap on smithi164 2024-09-10T03:21:57.119 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:57.201 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:56 smithi130 ceph-mon[23614]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:21:57.201 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:57.201 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:56 smithi130 ceph-mon[23614]: Reconfiguring mgr.smithi164.bkfwap (monmap changed)... 2024-09-10T03:21:57.201 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.bkfwap", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-10T03:21:57.201 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-10T03:21:57.201 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:57.201 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:56 smithi130 ceph-mon[23614]: Reconfiguring daemon mgr.smithi164.bkfwap on smithi164 2024-09-10T03:21:57.201 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:58.416 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:58 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:58.416 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:58 smithi164 ceph-mon[25711]: Reconfiguring mon.smithi164 (monmap changed)... 2024-09-10T03:21:58.416 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:58 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-10T03:21:58.416 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:58 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-10T03:21:58.416 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:58 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:58.416 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:58 smithi164 ceph-mon[25711]: Reconfiguring daemon mon.smithi164 on smithi164 2024-09-10T03:21:58.488 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:58 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:58.488 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:58 smithi130 ceph-mon[23614]: Reconfiguring mon.smithi164 (monmap changed)... 2024-09-10T03:21:58.488 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:58 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-10T03:21:58.488 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:58 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-10T03:21:58.488 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:58 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:58.488 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:58 smithi130 ceph-mon[23614]: Reconfiguring daemon mon.smithi164 on smithi164 2024-09-10T03:21:59.059 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:21:59.080 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph orch daemon add osd smithi130:vg_nvme/lv_3 2024-09-10T03:21:59.365 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:59 smithi130 ceph-mon[23614]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:21:59.365 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:59 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:59.365 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:59 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-10T03:21:59.366 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:59 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.130:9093"}]: dispatch 2024-09-10T03:21:59.366 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:59 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:59.366 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:59 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-10T03:21:59.366 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:59 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.130:3000"}]: dispatch 2024-09-10T03:21:59.366 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:59 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:59.366 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:59 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-10T03:21:59.366 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:59 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.130:9095"}]: dispatch 2024-09-10T03:21:59.366 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:59 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:59.366 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:59 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:21:59.366 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:59 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:59.366 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:21:59 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:21:59.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:59 smithi164 ceph-mon[25711]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:21:59.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:59 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:59.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:59 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-10T03:21:59.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:59 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.130:9093"}]: dispatch 2024-09-10T03:21:59.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:59 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:59.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:59 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-10T03:21:59.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:59 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.130:3000"}]: dispatch 2024-09-10T03:21:59.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:59 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:59.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:59 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-10T03:21:59.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:59 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.130:9095"}]: dispatch 2024-09-10T03:21:59.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:59 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:21:59.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:59 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:21:59.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:59 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:21:59.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:21:59 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:22:00.413 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:00 smithi130 ceph-mon[23614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-10T03:22:00.413 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:00 smithi130 ceph-mon[23614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.130:9093"}]: dispatch 2024-09-10T03:22:00.413 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:00 smithi130 ceph-mon[23614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-10T03:22:00.413 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:00 smithi130 ceph-mon[23614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.130:3000"}]: dispatch 2024-09-10T03:22:00.413 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:00 smithi130 ceph-mon[23614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-10T03:22:00.413 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:00 smithi130 ceph-mon[23614]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.130:9095"}]: dispatch 2024-09-10T03:22:00.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:00 smithi164 ceph-mon[25711]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-10T03:22:00.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:00 smithi164 ceph-mon[25711]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.130:9093"}]: dispatch 2024-09-10T03:22:00.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:00 smithi164 ceph-mon[25711]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-10T03:22:00.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:00 smithi164 ceph-mon[25711]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.130:3000"}]: dispatch 2024-09-10T03:22:00.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:00 smithi164 ceph-mon[25711]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-10T03:22:00.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:00 smithi164 ceph-mon[25711]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.130:9095"}]: dispatch 2024-09-10T03:22:01.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:01 smithi130 ceph-mon[23614]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:01.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:01 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:01.589 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:01 smithi164 ceph-mon[25711]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:01.589 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:01 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:03.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:03 smithi164 ceph-mon[25711]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:03.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:03 smithi164 ceph-mon[25711]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:22:03.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:03 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-10T03:22:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:03 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-10T03:22:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:03 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:22:03.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:03 smithi130 ceph-mon[23614]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:03.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:03 smithi130 ceph-mon[23614]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:22:03.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:03 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-10T03:22:03.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:03 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-10T03:22:03.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:03 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:22:05.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:05 smithi130 ceph-mon[23614]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:05.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:05 smithi164 ceph-mon[25711]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:06.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:06 smithi164 ceph-mon[25711]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:06.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:06 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:06.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:06 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/3145895824' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c18f93f0-590b-4928-94b5-0559db3cb557"}]: dispatch 2024-09-10T03:22:06.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:06 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/3145895824' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c18f93f0-590b-4928-94b5-0559db3cb557"}]': finished 2024-09-10T03:22:06.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:06 smithi164 ceph-mon[25711]: osdmap e11: 2 total, 1 up, 2 in 2024-09-10T03:22:06.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:06 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-10T03:22:07.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:06 smithi130 ceph-mon[23614]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:07.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:06 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:07.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:06 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3145895824' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c18f93f0-590b-4928-94b5-0559db3cb557"}]: dispatch 2024-09-10T03:22:07.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:06 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3145895824' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c18f93f0-590b-4928-94b5-0559db3cb557"}]': finished 2024-09-10T03:22:07.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:06 smithi130 ceph-mon[23614]: osdmap e11: 2 total, 1 up, 2 in 2024-09-10T03:22:07.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:06 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-10T03:22:07.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:07 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/385615720' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-10T03:22:08.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:07 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/385615720' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-10T03:22:08.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:08 smithi164 ceph-mon[25711]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:09.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:08 smithi130 ceph-mon[23614]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:10.657 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:10 smithi130 ceph-mon[23614]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:10.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:10 smithi164 ceph-mon[25711]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:13.078 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:13 smithi130 ceph-mon[23614]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:13.078 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:13 smithi130 ceph-mon[23614]: Detected new or changed devices on smithi130 2024-09-10T03:22:13.078 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:13 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:13.078 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:13 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:22:13.078 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:13 smithi130 ceph-mon[23614]: Adjusting osd_memory_target on smithi130 to 12864M 2024-09-10T03:22:13.078 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:13 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:13.078 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:13 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:13.078 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:13 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:22:13.078 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:13 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:22:13.078 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:13 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:22:13.426 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:13 smithi164 ceph-mon[25711]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:13.426 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:13 smithi164 ceph-mon[25711]: Detected new or changed devices on smithi130 2024-09-10T03:22:13.426 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:13 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:13.427 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:13 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:22:13.427 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:13 smithi164 ceph-mon[25711]: Adjusting osd_memory_target on smithi130 to 12864M 2024-09-10T03:22:13.427 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:13 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:13.427 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:13 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:13.427 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:13 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:22:13.427 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:13 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:22:13.427 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:13 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:22:14.372 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:14 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-10T03:22:14.372 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:14 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:22:14.372 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:14 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:14.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:14 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-10T03:22:14.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:14 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:22:14.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:14 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:15.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:15 smithi130 ceph-mon[23614]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:15.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:15 smithi130 ceph-mon[23614]: Deploying daemon osd.1 on smithi130 2024-09-10T03:22:15.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:15 smithi164 ceph-mon[25711]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:15.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:15 smithi164 ceph-mon[25711]: Deploying daemon osd.1 on smithi130 2024-09-10T03:22:17.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:17 smithi130 ceph-mon[23614]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:17.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:17 smithi164 ceph-mon[25711]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:19.197 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:19 smithi130 ceph-mon[23614]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:19.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:19 smithi164 ceph-mon[25711]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:20.708 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:20 smithi130 ceph-mon[23614]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:20.709 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:20 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:20.709 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:20 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:20.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:20 smithi164 ceph-mon[25711]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:20.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:20 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:20.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:20 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:22.195 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:21 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:22.195 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:21 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:22:22.195 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:21 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:22:22.195 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:21 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:22:22.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:21 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:22.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:21 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:22:22.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:21 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:22:22.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:21 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:22:22.887 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:22 smithi130 ceph-mon[23614]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:22.887 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:22 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:23.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:22 smithi164 ceph-mon[25711]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:23.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:22 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:24.423 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 1 on host 'smithi130' 2024-09-10T03:22:25.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:24 smithi130 ceph-mon[23614]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:25.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:24 smithi130 ceph-mon[23614]: from='osd.1 [v2:172.21.15.130:6810/1791466340,v1:172.21.15.130:6811/1791466340]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-10T03:22:25.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:24 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:25.087 DEBUG:teuthology.orchestra.run.smithi130:osd.1> sudo journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.1.service 2024-09-10T03:22:25.089 INFO:tasks.cephadm:Deploying osd.2 on smithi130 with /dev/vg_nvme/lv_2... 2024-09-10T03:22:25.089 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-10T03:22:25.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:24 smithi164 ceph-mon[25711]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:25.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:24 smithi164 ceph-mon[25711]: from='osd.1 [v2:172.21.15.130:6810/1791466340,v1:172.21.15.130:6811/1791466340]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-10T03:22:25.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:24 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:26.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:25 smithi164 ceph-mon[25711]: from='osd.1 [v2:172.21.15.130:6810/1791466340,v1:172.21.15.130:6811/1791466340]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-10T03:22:26.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:25 smithi164 ceph-mon[25711]: osdmap e12: 2 total, 1 up, 2 in 2024-09-10T03:22:26.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:25 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-10T03:22:26.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:25 smithi164 ceph-mon[25711]: from='osd.1 [v2:172.21.15.130:6810/1791466340,v1:172.21.15.130:6811/1791466340]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-09-10T03:22:26.238 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:25 smithi130 ceph-mon[23614]: from='osd.1 [v2:172.21.15.130:6810/1791466340,v1:172.21.15.130:6811/1791466340]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-10T03:22:26.238 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:25 smithi130 ceph-mon[23614]: osdmap e12: 2 total, 1 up, 2 in 2024-09-10T03:22:26.238 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:25 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-10T03:22:26.238 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:25 smithi130 ceph-mon[23614]: from='osd.1 [v2:172.21.15.130:6810/1791466340,v1:172.21.15.130:6811/1791466340]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-09-10T03:22:26.239 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:22:25 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1[38063]: 2024-09-10T03:22:25.862+0000 7fd543a75700 -1 osd.1 0 waiting for initial osdmap 2024-09-10T03:22:26.239 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:22:25 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1[38063]: 2024-09-10T03:22:25.865+0000 7fd53cc09700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-10T03:22:27.180 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:26 smithi130 ceph-mon[23614]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:27.180 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:26 smithi130 ceph-mon[23614]: from='osd.1 [v2:172.21.15.130:6810/1791466340,v1:172.21.15.130:6811/1791466340]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-09-10T03:22:27.181 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:26 smithi130 ceph-mon[23614]: osdmap e13: 2 total, 1 up, 2 in 2024-09-10T03:22:27.181 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:26 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-10T03:22:27.181 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:26 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-10T03:22:27.181 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:26 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-10T03:22:27.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:26 smithi164 ceph-mon[25711]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:27.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:26 smithi164 ceph-mon[25711]: from='osd.1 [v2:172.21.15.130:6810/1791466340,v1:172.21.15.130:6811/1791466340]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-09-10T03:22:27.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:26 smithi164 ceph-mon[25711]: osdmap e13: 2 total, 1 up, 2 in 2024-09-10T03:22:27.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:26 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-10T03:22:27.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:26 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-10T03:22:27.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:26 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-10T03:22:28.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:27 smithi164 ceph-mon[25711]: purged_snaps scrub starts 2024-09-10T03:22:28.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:27 smithi164 ceph-mon[25711]: purged_snaps scrub ok 2024-09-10T03:22:28.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:27 smithi164 ceph-mon[25711]: osd.1 [v2:172.21.15.130:6810/1791466340,v1:172.21.15.130:6811/1791466340] boot 2024-09-10T03:22:28.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:27 smithi164 ceph-mon[25711]: osdmap e14: 2 total, 2 up, 2 in 2024-09-10T03:22:28.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:27 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-10T03:22:28.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:27 smithi130 ceph-mon[23614]: purged_snaps scrub starts 2024-09-10T03:22:28.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:27 smithi130 ceph-mon[23614]: purged_snaps scrub ok 2024-09-10T03:22:28.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:27 smithi130 ceph-mon[23614]: osd.1 [v2:172.21.15.130:6810/1791466340,v1:172.21.15.130:6811/1791466340] boot 2024-09-10T03:22:28.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:27 smithi130 ceph-mon[23614]: osdmap e14: 2 total, 2 up, 2 in 2024-09-10T03:22:28.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:27 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-10T03:22:28.270 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:22:28.291 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph orch daemon add osd smithi130:vg_nvme/lv_2 2024-09-10T03:22:29.131 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:28 smithi130 ceph-mon[23614]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:29.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:28 smithi164 ceph-mon[25711]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-10T03:22:30.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:29 smithi164 ceph-mon[25711]: osdmap e15: 2 total, 2 up, 2 in 2024-09-10T03:22:30.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:29 smithi130 ceph-mon[23614]: osdmap e15: 2 total, 2 up, 2 in 2024-09-10T03:22:30.909 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:30 smithi130 ceph-mon[23614]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:31.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:30 smithi164 ceph-mon[25711]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:32.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:31 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:32.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:31 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-10T03:22:32.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:31 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-10T03:22:32.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:31 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:22:32.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:31 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:32.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:31 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-10T03:22:32.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:31 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-10T03:22:32.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:31 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:22:33.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:32 smithi164 ceph-mon[25711]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:22:33.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:32 smithi164 ceph-mon[25711]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:33.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:32 smithi130 ceph-mon[23614]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:22:33.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:32 smithi130 ceph-mon[23614]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:35.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:34 smithi164 ceph-mon[25711]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:35.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:35.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:22:35.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:22:35.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:35.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:35.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:22:35.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:22:35.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:22:35.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:34 smithi130 ceph-mon[23614]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:35.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:35.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:22:35.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:22:35.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:35.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:35.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:22:35.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:22:35.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:22:36.250 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:35 smithi130 ceph-mon[23614]: Adjusting osd_memory_target on smithi130 to 6432M 2024-09-10T03:22:36.250 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:35 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/935925936' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "78696ee3-3b0f-4c43-8f58-c9fd1a429c65"}]: dispatch 2024-09-10T03:22:36.250 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:35 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/935925936' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "78696ee3-3b0f-4c43-8f58-c9fd1a429c65"}]': finished 2024-09-10T03:22:36.251 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:35 smithi130 ceph-mon[23614]: osdmap e16: 3 total, 2 up, 3 in 2024-09-10T03:22:36.251 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-10T03:22:36.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:35 smithi164 ceph-mon[25711]: Adjusting osd_memory_target on smithi130 to 6432M 2024-09-10T03:22:36.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:35 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/935925936' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "78696ee3-3b0f-4c43-8f58-c9fd1a429c65"}]: dispatch 2024-09-10T03:22:36.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:35 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/935925936' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "78696ee3-3b0f-4c43-8f58-c9fd1a429c65"}]': finished 2024-09-10T03:22:36.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:35 smithi164 ceph-mon[25711]: osdmap e16: 3 total, 2 up, 3 in 2024-09-10T03:22:36.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-10T03:22:37.217 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:36 smithi130 ceph-mon[23614]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:37.218 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:36 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3253467526' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-10T03:22:37.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:36 smithi164 ceph-mon[25711]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:37.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:36 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/3253467526' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-10T03:22:39.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:38 smithi130 ceph-mon[23614]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:39.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:38 smithi164 ceph-mon[25711]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:40.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:40 smithi164 ceph-mon[25711]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:40.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:40 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:40.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:40 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:40.961 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:40 smithi130 ceph-mon[23614]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:40.961 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:40 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:40.961 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:40 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:41.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:41 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:22:41.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:41 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:22:41.984 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:41 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:22:41.984 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:41 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:22:42.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:42 smithi164 ceph-mon[25711]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:42.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-10T03:22:42.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:22:42.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:42 smithi164 ceph-mon[25711]: Deploying daemon osd.2 on smithi130 2024-09-10T03:22:42.987 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:42 smithi130 ceph-mon[23614]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:42.987 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-10T03:22:42.987 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:22:42.987 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:42 smithi130 ceph-mon[23614]: Deploying daemon osd.2 on smithi130 2024-09-10T03:22:44.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:44 smithi164 ceph-mon[25711]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:44.973 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:44 smithi130 ceph-mon[23614]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:46.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:46 smithi164 ceph-mon[25711]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:47.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:46 smithi130 ceph-mon[23614]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:48.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:48 smithi164 ceph-mon[25711]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:48.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:48 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:48.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:48 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:22:48.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:48 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:22:48.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:48 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:22:49.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:48 smithi130 ceph-mon[23614]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:49.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:48 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:49.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:48 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:22:49.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:48 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:22:49.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:48 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:22:50.710 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:50 smithi130 ceph-mon[23614]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:50.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:50 smithi164 ceph-mon[25711]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:51.132 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 2 on host 'smithi130' 2024-09-10T03:22:51.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:51 smithi130 ceph-mon[23614]: from='osd.2 [v2:172.21.15.130:6818/2802507353,v1:172.21.15.130:6819/2802507353]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-10T03:22:51.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:51 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:51.837 DEBUG:teuthology.orchestra.run.smithi130:osd.2> sudo journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.2.service 2024-09-10T03:22:51.840 INFO:tasks.cephadm:Deploying osd.3 on smithi130 with /dev/vg_nvme/lv_1... 2024-09-10T03:22:51.840 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-10T03:22:51.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:51 smithi164 ceph-mon[25711]: from='osd.2 [v2:172.21.15.130:6818/2802507353,v1:172.21.15.130:6819/2802507353]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-10T03:22:51.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:51 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:52.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:52 smithi164 ceph-mon[25711]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:52.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:52 smithi164 ceph-mon[25711]: from='osd.2 [v2:172.21.15.130:6818/2802507353,v1:172.21.15.130:6819/2802507353]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-10T03:22:52.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:52 smithi164 ceph-mon[25711]: osdmap e17: 3 total, 2 up, 3 in 2024-09-10T03:22:52.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:52 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-10T03:22:52.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:52 smithi164 ceph-mon[25711]: from='osd.2 [v2:172.21.15.130:6818/2802507353,v1:172.21.15.130:6819/2802507353]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-09-10T03:22:53.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:52 smithi130 ceph-mon[23614]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:53.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:52 smithi130 ceph-mon[23614]: from='osd.2 [v2:172.21.15.130:6818/2802507353,v1:172.21.15.130:6819/2802507353]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-10T03:22:53.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:52 smithi130 ceph-mon[23614]: osdmap e17: 3 total, 2 up, 3 in 2024-09-10T03:22:53.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:52 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-10T03:22:53.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:52 smithi130 ceph-mon[23614]: from='osd.2 [v2:172.21.15.130:6818/2802507353,v1:172.21.15.130:6819/2802507353]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-09-10T03:22:53.013 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:22:52 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2[41711]: 2024-09-10T03:22:52.634+0000 7f3873e3e700 -1 osd.2 0 waiting for initial osdmap 2024-09-10T03:22:53.013 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:22:52 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2[41711]: 2024-09-10T03:22:52.641+0000 7f386f7d7700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-10T03:22:53.906 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:53 smithi130 ceph-mon[23614]: from='osd.2 [v2:172.21.15.130:6818/2802507353,v1:172.21.15.130:6819/2802507353]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-09-10T03:22:53.906 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:53 smithi130 ceph-mon[23614]: osdmap e18: 3 total, 2 up, 3 in 2024-09-10T03:22:53.906 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:53 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-10T03:22:53.906 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:53 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-10T03:22:53.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:53 smithi164 ceph-mon[25711]: from='osd.2 [v2:172.21.15.130:6818/2802507353,v1:172.21.15.130:6819/2802507353]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-09-10T03:22:53.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:53 smithi164 ceph-mon[25711]: osdmap e18: 3 total, 2 up, 3 in 2024-09-10T03:22:53.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:53 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-10T03:22:53.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:53 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-10T03:22:54.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:54 smithi164 ceph-mon[25711]: purged_snaps scrub starts 2024-09-10T03:22:54.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:54 smithi164 ceph-mon[25711]: purged_snaps scrub ok 2024-09-10T03:22:54.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:54 smithi164 ceph-mon[25711]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:54.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:54 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-10T03:22:54.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:54 smithi164 ceph-mon[25711]: osd.2 [v2:172.21.15.130:6818/2802507353,v1:172.21.15.130:6819/2802507353] boot 2024-09-10T03:22:54.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:54 smithi164 ceph-mon[25711]: osdmap e19: 3 total, 3 up, 3 in 2024-09-10T03:22:54.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:54 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-10T03:22:55.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:54 smithi130 ceph-mon[23614]: purged_snaps scrub starts 2024-09-10T03:22:55.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:54 smithi130 ceph-mon[23614]: purged_snaps scrub ok 2024-09-10T03:22:55.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:54 smithi130 ceph-mon[23614]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-10T03:22:55.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:54 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-10T03:22:55.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:54 smithi130 ceph-mon[23614]: osd.2 [v2:172.21.15.130:6818/2802507353,v1:172.21.15.130:6819/2802507353] boot 2024-09-10T03:22:55.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:54 smithi130 ceph-mon[23614]: osdmap e19: 3 total, 3 up, 3 in 2024-09-10T03:22:55.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:54 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-10T03:22:55.382 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:22:55.400 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph orch daemon add osd smithi130:vg_nvme/lv_1 2024-09-10T03:22:56.132 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:55 smithi130 ceph-mon[23614]: osdmap e20: 3 total, 3 up, 3 in 2024-09-10T03:22:56.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:55 smithi164 ceph-mon[25711]: osdmap e20: 3 total, 3 up, 3 in 2024-09-10T03:22:57.205 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:56 smithi130 ceph-mon[23614]: pgmap v75: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-10T03:22:57.205 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-10T03:22:57.205 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-10T03:22:57.205 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:56 smithi130 ceph-mon[23614]: osdmap e21: 3 total, 3 up, 3 in 2024-09-10T03:22:57.206 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-10T03:22:57.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:56 smithi164 ceph-mon[25711]: pgmap v75: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-10T03:22:57.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-10T03:22:57.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-10T03:22:57.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:56 smithi164 ceph-mon[25711]: osdmap e21: 3 total, 3 up, 3 in 2024-09-10T03:22:57.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-10T03:22:58.067 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:57 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-10T03:22:58.067 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:57 smithi130 ceph-mon[23614]: osdmap e22: 3 total, 3 up, 3 in 2024-09-10T03:22:58.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:57 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-10T03:22:58.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:57 smithi164 ceph-mon[25711]: osdmap e22: 3 total, 3 up, 3 in 2024-09-10T03:22:58.327 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:22:58 smithi130 sudo[43184]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-10T03:22:58.327 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:22:58 smithi130 sudo[43184]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-10T03:22:58.327 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:22:58 smithi130 sudo[43184]: pam_unix(sudo:session): session closed for user root 2024-09-10T03:22:58.577 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:22:58 smithi130 sudo[43268]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-10T03:22:58.577 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:22:58 smithi130 sudo[43268]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-10T03:22:58.577 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:22:58 smithi130 sudo[43268]: pam_unix(sudo:session): session closed for user root 2024-09-10T03:22:58.919 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:22:58 smithi130 sudo[43300]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-10T03:22:58.919 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:22:58 smithi130 sudo[43300]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-10T03:22:58.919 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:22:58 smithi130 sudo[43300]: pam_unix(sudo:session): session closed for user root 2024-09-10T03:22:59.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:58 smithi164 ceph-mon[25711]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-10T03:22:59.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:58 smithi164 ceph-mon[25711]: osdmap e23: 3 total, 3 up, 3 in 2024-09-10T03:22:59.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:58 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:59.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:58 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-10T03:22:59.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:58 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-10T03:22:59.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:58 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:22:59.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:22:58 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:59.248 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:22:59 smithi130 sudo[43316]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-10T03:22:59.249 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:22:59 smithi130 sudo[43316]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-10T03:22:59.249 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:22:59 smithi130 sudo[43316]: pam_unix(sudo:session): session closed for user root 2024-09-10T03:22:59.249 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:58 smithi130 ceph-mon[23614]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-10T03:22:59.249 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:58 smithi130 ceph-mon[23614]: osdmap e23: 3 total, 3 up, 3 in 2024-09-10T03:22:59.249 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:58 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:59.249 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:58 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-10T03:22:59.249 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:58 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-10T03:22:59.249 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:58 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:22:59.249 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:58 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:22:59.511 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:22:59 smithi130 sudo[43321]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-10T03:22:59.511 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:22:59 smithi130 sudo[43321]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-10T03:22:59.511 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:22:59 smithi130 sudo[43321]: pam_unix(sudo:session): session closed for user root 2024-09-10T03:22:59.763 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:22:59 smithi130 sudo[43343]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-10T03:22:59.763 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:22:59 smithi130 sudo[43343]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-10T03:22:59.763 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:22:59 smithi130 sudo[43343]: pam_unix(sudo:session): session closed for user root 2024-09-10T03:23:00.035 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:59 smithi130 sudo[43354]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-10T03:23:00.035 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:59 smithi130 sudo[43354]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-10T03:23:00.035 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:22:59 smithi130 sudo[43354]: pam_unix(sudo:session): session closed for user root 2024-09-10T03:23:00.301 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:00 smithi164 ceph-mon[25711]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:23:00.301 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:00 smithi164 sudo[28027]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-10T03:23:00.301 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:00 smithi164 sudo[28027]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-10T03:23:00.301 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:00 smithi164 sudo[28027]: pam_unix(sudo:session): session closed for user root 2024-09-10T03:23:00.390 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:00 smithi130 sudo[43381]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-10T03:23:00.390 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:00 smithi130 sudo[43381]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-10T03:23:00.390 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:00 smithi130 sudo[43381]: pam_unix(sudo:session): session closed for user root 2024-09-10T03:23:00.390 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:00 smithi130 ceph-mon[23614]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:23:00.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:00 smithi164 sudo[28030]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-10T03:23:00.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:00 smithi164 sudo[28030]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-10T03:23:00.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:00 smithi164 sudo[28030]: pam_unix(sudo:session): session closed for user root 2024-09-10T03:23:01.215 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:01 smithi130 ceph-mon[23614]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-10T03:23:01.215 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:01 smithi130 ceph-mon[23614]: pgmap v80: 1 pgs: 1 active+clean; 128 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-10T03:23:01.215 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:01 smithi130 ceph-mon[23614]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-10T03:23:01.215 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:01 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi130"}]: dispatch 2024-09-10T03:23:01.215 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:01 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:23:01.215 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:01 smithi130 ceph-mon[23614]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-10T03:23:01.215 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:01 smithi130 ceph-mon[23614]: mgrmap e19: smithi130.axcmel(active, since 2m), standbys: smithi164.bkfwap 2024-09-10T03:23:01.215 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:01 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi130"}]: dispatch 2024-09-10T03:23:01.215 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:01 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:23:01.215 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:01 smithi130 ceph-mon[23614]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-10T03:23:01.215 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:01 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:01.390 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:01 smithi164 ceph-mon[25711]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-10T03:23:01.390 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:01 smithi164 ceph-mon[25711]: pgmap v80: 1 pgs: 1 active+clean; 128 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-10T03:23:01.390 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:01 smithi164 ceph-mon[25711]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-10T03:23:01.390 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:01 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi130"}]: dispatch 2024-09-10T03:23:01.390 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:01 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:23:01.390 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:01 smithi164 ceph-mon[25711]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-10T03:23:01.390 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:01 smithi164 ceph-mon[25711]: mgrmap e19: smithi130.axcmel(active, since 2m), standbys: smithi164.bkfwap 2024-09-10T03:23:01.390 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:01 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi130"}]: dispatch 2024-09-10T03:23:01.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:01 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:23:01.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:01 smithi164 ceph-mon[25711]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-10T03:23:01.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:01 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:02.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:02 smithi164 ceph-mon[25711]: Detected new or changed devices on smithi130 2024-09-10T03:23:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:02 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:02 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:23:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:02 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:23:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:02 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:23:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:02 smithi164 ceph-mon[25711]: Adjusting osd_memory_target on smithi130 to 4288M 2024-09-10T03:23:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:02 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:02 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:02 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:23:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:02 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:02 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:23:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:02 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:02.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:02 smithi130 ceph-mon[23614]: Detected new or changed devices on smithi130 2024-09-10T03:23:02.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:02 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:02.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:02 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:23:02.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:02 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:23:02.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:02 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:23:02.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:02 smithi130 ceph-mon[23614]: Adjusting osd_memory_target on smithi130 to 4288M 2024-09-10T03:23:02.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:02 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:02.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:02 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:02.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:02 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:23:02.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:02 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:02.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:02 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:23:02.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:02 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:03.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:03 smithi164 ceph-mon[25711]: pgmap v81: 1 pgs: 1 active+clean; 128 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-10T03:23:03.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:03 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/2757480600' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1538e485-8857-450d-87f3-1ab837a97795"}]: dispatch 2024-09-10T03:23:03.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:03 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/2757480600' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1538e485-8857-450d-87f3-1ab837a97795"}]': finished 2024-09-10T03:23:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:03 smithi164 ceph-mon[25711]: osdmap e24: 4 total, 3 up, 4 in 2024-09-10T03:23:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:03 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-10T03:23:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:03 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/176136174' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-10T03:23:03.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:03 smithi130 ceph-mon[23614]: pgmap v81: 1 pgs: 1 active+clean; 128 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-10T03:23:03.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:03 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/2757480600' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1538e485-8857-450d-87f3-1ab837a97795"}]: dispatch 2024-09-10T03:23:03.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:03 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/2757480600' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1538e485-8857-450d-87f3-1ab837a97795"}]': finished 2024-09-10T03:23:03.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:03 smithi130 ceph-mon[23614]: osdmap e24: 4 total, 3 up, 4 in 2024-09-10T03:23:03.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:03 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-10T03:23:03.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:03 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/176136174' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-10T03:23:05.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:05 smithi164 ceph-mon[25711]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 14 KiB/s rd, 404 KiB/s wr, 38 op/s 2024-09-10T03:23:05.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:05 smithi130 ceph-mon[23614]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 14 KiB/s rd, 404 KiB/s wr, 38 op/s 2024-09-10T03:23:07.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:07 smithi130 ceph-mon[23614]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 11 KiB/s rd, 299 KiB/s wr, 28 op/s 2024-09-10T03:23:07.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:07 smithi164 ceph-mon[25711]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 11 KiB/s rd, 299 KiB/s wr, 28 op/s 2024-09-10T03:23:09.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:09 smithi130 ceph-mon[23614]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 8.4 KiB/s rd, 237 KiB/s wr, 22 op/s 2024-09-10T03:23:09.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:09 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-10T03:23:09.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:09 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:09.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:09 smithi164 ceph-mon[25711]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 8.4 KiB/s rd, 237 KiB/s wr, 22 op/s 2024-09-10T03:23:09.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:09 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-10T03:23:09.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:09 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:10.155 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:10 smithi130 ceph-mon[23614]: Deploying daemon osd.3 on smithi130 2024-09-10T03:23:10.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:10 smithi164 ceph-mon[25711]: Deploying daemon osd.3 on smithi130 2024-09-10T03:23:11.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:11 smithi130 ceph-mon[23614]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 8.1 KiB/s rd, 229 KiB/s wr, 21 op/s 2024-09-10T03:23:11.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:11 smithi164 ceph-mon[25711]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 8.1 KiB/s rd, 229 KiB/s wr, 21 op/s 2024-09-10T03:23:13.369 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:13 smithi130 ceph-mon[23614]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 8.1 KiB/s rd, 229 KiB/s wr, 21 op/s 2024-09-10T03:23:13.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:13 smithi164 ceph-mon[25711]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 8.1 KiB/s rd, 229 KiB/s wr, 21 op/s 2024-09-10T03:23:14.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:14 smithi164 ceph-mon[25711]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 7.3 KiB/s rd, 205 KiB/s wr, 19 op/s 2024-09-10T03:23:14.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:14 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:14.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:14 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:23:14.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:14 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:14.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:14 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:23:15.010 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:14 smithi130 ceph-mon[23614]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 7.3 KiB/s rd, 205 KiB/s wr, 19 op/s 2024-09-10T03:23:15.010 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:14 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:15.011 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:14 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:23:15.011 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:14 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:15.011 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:14 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:23:17.417 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:17 smithi130 ceph-mon[23614]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-10T03:23:17.417 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:17 smithi130 ceph-mon[23614]: from='osd.3 [v2:172.21.15.130:6826/3722528714,v1:172.21.15.130:6827/3722528714]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-10T03:23:17.448 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 3 on host 'smithi130' 2024-09-10T03:23:17.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:17 smithi164 ceph-mon[25711]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-10T03:23:17.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:17 smithi164 ceph-mon[25711]: from='osd.3 [v2:172.21.15.130:6826/3722528714,v1:172.21.15.130:6827/3722528714]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-10T03:23:18.012 DEBUG:teuthology.orchestra.run.smithi130:osd.3> sudo journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.3.service 2024-09-10T03:23:18.015 INFO:tasks.cephadm:Deploying osd.4 on smithi164 with /dev/vg_nvme/lv_4... 2024-09-10T03:23:18.015 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-10T03:23:18.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:18 smithi130 ceph-mon[23614]: from='osd.3 [v2:172.21.15.130:6826/3722528714,v1:172.21.15.130:6827/3722528714]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-10T03:23:18.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:18 smithi130 ceph-mon[23614]: osdmap e25: 4 total, 3 up, 4 in 2024-09-10T03:23:18.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:18 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-10T03:23:18.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:18 smithi130 ceph-mon[23614]: from='osd.3 [v2:172.21.15.130:6826/3722528714,v1:172.21.15.130:6827/3722528714]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-09-10T03:23:18.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:18 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:18.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:18 smithi130 ceph-mon[23614]: from='osd.3 [v2:172.21.15.130:6826/3722528714,v1:172.21.15.130:6827/3722528714]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-09-10T03:23:18.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:18 smithi130 ceph-mon[23614]: osdmap e26: 4 total, 3 up, 4 in 2024-09-10T03:23:18.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:18 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-10T03:23:18.263 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:23:18 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3[44936]: 2024-09-10T03:23:18.054+0000 7f17a0736700 -1 osd.3 0 waiting for initial osdmap 2024-09-10T03:23:18.264 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:23:18 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3[44936]: 2024-09-10T03:23:18.061+0000 7f179d0d1700 -1 osd.3 26 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-10T03:23:18.326 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:18 smithi164 ceph-mon[25711]: from='osd.3 [v2:172.21.15.130:6826/3722528714,v1:172.21.15.130:6827/3722528714]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-10T03:23:18.326 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:18 smithi164 ceph-mon[25711]: osdmap e25: 4 total, 3 up, 4 in 2024-09-10T03:23:18.326 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:18 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-10T03:23:18.326 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:18 smithi164 ceph-mon[25711]: from='osd.3 [v2:172.21.15.130:6826/3722528714,v1:172.21.15.130:6827/3722528714]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-09-10T03:23:18.326 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:18 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:18.326 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:18 smithi164 ceph-mon[25711]: from='osd.3 [v2:172.21.15.130:6826/3722528714,v1:172.21.15.130:6827/3722528714]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-09-10T03:23:18.326 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:18 smithi164 ceph-mon[25711]: osdmap e26: 4 total, 3 up, 4 in 2024-09-10T03:23:18.326 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:18 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-10T03:23:19.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:19 smithi164 ceph-mon[25711]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-10T03:23:19.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:19 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-10T03:23:19.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:19 smithi130 ceph-mon[23614]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-10T03:23:19.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:19 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-10T03:23:20.332 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:20 smithi164 ceph-mon[25711]: purged_snaps scrub starts 2024-09-10T03:23:20.333 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:20 smithi164 ceph-mon[25711]: purged_snaps scrub ok 2024-09-10T03:23:20.333 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:20 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-10T03:23:20.333 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:20 smithi164 ceph-mon[25711]: osd.3 [v2:172.21.15.130:6826/3722528714,v1:172.21.15.130:6827/3722528714] boot 2024-09-10T03:23:20.333 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:20 smithi164 ceph-mon[25711]: osdmap e27: 4 total, 4 up, 4 in 2024-09-10T03:23:20.333 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:20 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-10T03:23:20.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:20 smithi130 ceph-mon[23614]: purged_snaps scrub starts 2024-09-10T03:23:20.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:20 smithi130 ceph-mon[23614]: purged_snaps scrub ok 2024-09-10T03:23:20.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:20 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-10T03:23:20.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:20 smithi130 ceph-mon[23614]: osd.3 [v2:172.21.15.130:6826/3722528714,v1:172.21.15.130:6827/3722528714] boot 2024-09-10T03:23:20.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:20 smithi130 ceph-mon[23614]: osdmap e27: 4 total, 4 up, 4 in 2024-09-10T03:23:20.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:20 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-10T03:23:20.855 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:23:20.874 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph orch daemon add osd smithi164:vg_nvme/lv_4 2024-09-10T03:23:21.180 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:21 smithi164 ceph-mon[25711]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-10T03:23:21.180 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:21 smithi164 ceph-mon[25711]: osdmap e28: 4 total, 4 up, 4 in 2024-09-10T03:23:21.180 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:21 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:21.191 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:21 smithi130 ceph-mon[23614]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-10T03:23:21.191 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:21 smithi130 ceph-mon[23614]: osdmap e28: 4 total, 4 up, 4 in 2024-09-10T03:23:21.191 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:21 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:23.151 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:23 smithi164 ceph-mon[25711]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:23.151 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:23 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-10T03:23:23.151 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:23 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-10T03:23:23.151 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:23 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:23.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:23 smithi130 ceph-mon[23614]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:23.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:23 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-10T03:23:23.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:23 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-10T03:23:23.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:23 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:24.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:24 smithi164 ceph-mon[25711]: from='client.24141 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:23:24.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:24 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:24.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:24 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:23:24.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:24 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:23:24.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:24 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:23:24.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:24 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:23:24.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:24 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:24.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:24 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:24.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:24 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:23:24.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:24 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:24.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:24 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:23:24.098 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:24 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:24.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:24 smithi130 ceph-mon[23614]: from='client.24141 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:23:24.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:24 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:24.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:24 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:23:24.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:24 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:23:24.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:24 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:23:24.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:24 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:23:24.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:24 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:24.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:24 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:24.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:24 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:23:24.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:24 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:24.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:24 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:23:24.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:24 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:25.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:25 smithi164 ceph-mon[25711]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:25.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:25 smithi164 ceph-mon[25711]: Detected new or changed devices on smithi130 2024-09-10T03:23:25.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:25 smithi164 ceph-mon[25711]: Adjusting osd_memory_target on smithi130 to 3216M 2024-09-10T03:23:25.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:25 smithi164 ceph-mon[25711]: from='client.? 172.21.15.164:0/86980666' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a381f70a-1f9a-4cab-9888-5e8f14e67528"}]: dispatch 2024-09-10T03:23:25.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:25 smithi164 ceph-mon[25711]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a381f70a-1f9a-4cab-9888-5e8f14e67528"}]: dispatch 2024-09-10T03:23:25.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:25 smithi164 ceph-mon[25711]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a381f70a-1f9a-4cab-9888-5e8f14e67528"}]': finished 2024-09-10T03:23:25.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:25 smithi164 ceph-mon[25711]: osdmap e29: 5 total, 4 up, 5 in 2024-09-10T03:23:25.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:25 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-10T03:23:25.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:25 smithi130 ceph-mon[23614]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:25.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:25 smithi130 ceph-mon[23614]: Detected new or changed devices on smithi130 2024-09-10T03:23:25.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:25 smithi130 ceph-mon[23614]: Adjusting osd_memory_target on smithi130 to 3216M 2024-09-10T03:23:25.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:25 smithi130 ceph-mon[23614]: from='client.? 172.21.15.164:0/86980666' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a381f70a-1f9a-4cab-9888-5e8f14e67528"}]: dispatch 2024-09-10T03:23:25.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:25 smithi130 ceph-mon[23614]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a381f70a-1f9a-4cab-9888-5e8f14e67528"}]: dispatch 2024-09-10T03:23:25.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:25 smithi130 ceph-mon[23614]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a381f70a-1f9a-4cab-9888-5e8f14e67528"}]': finished 2024-09-10T03:23:25.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:25 smithi130 ceph-mon[23614]: osdmap e29: 5 total, 4 up, 5 in 2024-09-10T03:23:25.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:25 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-10T03:23:26.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:26 smithi164 ceph-mon[25711]: from='client.? 172.21.15.164:0/3108384801' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-10T03:23:26.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:26 smithi130 ceph-mon[23614]: from='client.? 172.21.15.164:0/3108384801' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-10T03:23:27.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:27 smithi164 ceph-mon[25711]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:27.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:27 smithi130 ceph-mon[23614]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:29.300 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:29 smithi164 ceph-mon[25711]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:29.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:29 smithi130 ceph-mon[23614]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:31.356 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:31 smithi164 ceph-mon[25711]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:31.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:31 smithi130 ceph-mon[23614]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:32.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:32 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-10T03:23:32.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:32 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:32.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:32 smithi164 ceph-mon[25711]: Deploying daemon osd.4 on smithi164 2024-09-10T03:23:32.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:32 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-10T03:23:32.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:32 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:32.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:32 smithi130 ceph-mon[23614]: Deploying daemon osd.4 on smithi164 2024-09-10T03:23:33.422 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:33 smithi164 ceph-mon[25711]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:33.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:33 smithi130 ceph-mon[23614]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:35.312 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:35 smithi164 ceph-mon[25711]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:35.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:35 smithi130 ceph-mon[23614]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:37.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:37 smithi164 ceph-mon[25711]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:37.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:37 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:37.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:37 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:23:37.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:37 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:37.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:37 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:23:37.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:37 smithi130 ceph-mon[23614]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:37.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:37 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:37.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:37 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:23:37.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:37 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:37.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:37 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:23:39.163 INFO:teuthology.orchestra.run.smithi164.stdout:Created osd(s) 4 on host 'smithi164' 2024-09-10T03:23:39.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:39 smithi164 ceph-mon[25711]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:39.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:39 smithi130 ceph-mon[23614]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:39.873 DEBUG:teuthology.orchestra.run.smithi164:osd.4> sudo journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.4.service 2024-09-10T03:23:39.876 INFO:tasks.cephadm:Deploying osd.5 on smithi164 with /dev/vg_nvme/lv_3... 2024-09-10T03:23:39.876 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-10T03:23:40.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:40 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:40.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:40 smithi164 ceph-mon[25711]: from='osd.4 [v2:172.21.15.164:6800/1249950658,v1:172.21.15.164:6801/1249950658]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-10T03:23:40.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:40 smithi164 ceph-mon[25711]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-10T03:23:40.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:40 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:40.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:40 smithi130 ceph-mon[23614]: from='osd.4 [v2:172.21.15.164:6800/1249950658,v1:172.21.15.164:6801/1249950658]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-10T03:23:40.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:40 smithi130 ceph-mon[23614]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-10T03:23:41.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:41 smithi164 ceph-mon[25711]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:41.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:41 smithi164 ceph-mon[25711]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-10T03:23:41.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:41 smithi164 ceph-mon[25711]: osdmap e30: 5 total, 4 up, 5 in 2024-09-10T03:23:41.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:41 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-10T03:23:41.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:41 smithi164 ceph-mon[25711]: from='osd.4 [v2:172.21.15.164:6800/1249950658,v1:172.21.15.164:6801/1249950658]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-09-10T03:23:41.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:41 smithi164 ceph-mon[25711]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-09-10T03:23:41.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:41 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:41.458 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:23:41 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[29939]: 2024-09-10T03:23:41.171+0000 7f018b4da700 -1 osd.4 0 waiting for initial osdmap 2024-09-10T03:23:41.458 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:23:41 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[29939]: 2024-09-10T03:23:41.175+0000 7f0187e75700 -1 osd.4 31 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-10T03:23:41.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:41 smithi130 ceph-mon[23614]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:41.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:41 smithi130 ceph-mon[23614]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-10T03:23:41.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:41 smithi130 ceph-mon[23614]: osdmap e30: 5 total, 4 up, 5 in 2024-09-10T03:23:41.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:41 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-10T03:23:41.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:41 smithi130 ceph-mon[23614]: from='osd.4 [v2:172.21.15.164:6800/1249950658,v1:172.21.15.164:6801/1249950658]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-09-10T03:23:41.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:41 smithi130 ceph-mon[23614]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-09-10T03:23:41.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:41 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:42.449 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:42 smithi164 ceph-mon[25711]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2024-09-10T03:23:42.497 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:42 smithi164 ceph-mon[25711]: osdmap e31: 5 total, 4 up, 5 in 2024-09-10T03:23:42.497 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-10T03:23:42.497 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-10T03:23:42.497 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:23:42.497 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:23:42.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:42 smithi130 ceph-mon[23614]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2024-09-10T03:23:42.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:42 smithi130 ceph-mon[23614]: osdmap e31: 5 total, 4 up, 5 in 2024-09-10T03:23:42.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-10T03:23:42.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-10T03:23:42.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:23:42.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:23:43.387 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:43 smithi164 ceph-mon[25711]: purged_snaps scrub starts 2024-09-10T03:23:43.387 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:43 smithi164 ceph-mon[25711]: purged_snaps scrub ok 2024-09-10T03:23:43.387 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:43 smithi164 ceph-mon[25711]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:43.388 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:43 smithi164 ceph-mon[25711]: osd.4 [v2:172.21.15.164:6800/1249950658,v1:172.21.15.164:6801/1249950658] boot 2024-09-10T03:23:43.388 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:43 smithi164 ceph-mon[25711]: osdmap e32: 5 total, 5 up, 5 in 2024-09-10T03:23:43.388 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:43 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-10T03:23:43.388 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:43 smithi164 ceph-mon[25711]: osdmap e33: 5 total, 5 up, 5 in 2024-09-10T03:23:43.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:43 smithi130 ceph-mon[23614]: purged_snaps scrub starts 2024-09-10T03:23:43.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:43 smithi130 ceph-mon[23614]: purged_snaps scrub ok 2024-09-10T03:23:43.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:43 smithi130 ceph-mon[23614]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-10T03:23:43.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:43 smithi130 ceph-mon[23614]: osd.4 [v2:172.21.15.164:6800/1249950658,v1:172.21.15.164:6801/1249950658] boot 2024-09-10T03:23:43.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:43 smithi130 ceph-mon[23614]: osdmap e32: 5 total, 5 up, 5 in 2024-09-10T03:23:43.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:43 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-10T03:23:43.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:43 smithi130 ceph-mon[23614]: osdmap e33: 5 total, 5 up, 5 in 2024-09-10T03:23:43.971 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:23:43.993 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph orch daemon add osd smithi164:vg_nvme/lv_3 2024-09-10T03:23:45.438 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:45 smithi164 ceph-mon[25711]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-10T03:23:45.439 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:45 smithi164 ceph-mon[25711]: osdmap e34: 5 total, 5 up, 5 in 2024-09-10T03:23:45.439 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:45 smithi164 ceph-mon[25711]: osdmap e35: 5 total, 5 up, 5 in 2024-09-10T03:23:45.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:45 smithi130 ceph-mon[23614]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-10T03:23:45.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:45 smithi130 ceph-mon[23614]: osdmap e34: 5 total, 5 up, 5 in 2024-09-10T03:23:45.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:45 smithi130 ceph-mon[23614]: osdmap e35: 5 total, 5 up, 5 in 2024-09-10T03:23:46.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:46 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-10T03:23:46.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:46 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-10T03:23:46.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:46 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:46.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:46 smithi164 ceph-mon[25711]: osdmap e36: 5 total, 5 up, 5 in 2024-09-10T03:23:46.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:46 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-10T03:23:46.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:46 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-10T03:23:46.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:46 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:46.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:46 smithi130 ceph-mon[23614]: osdmap e36: 5 total, 5 up, 5 in 2024-09-10T03:23:47.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:47 smithi164 ceph-mon[25711]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-10T03:23:47.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:47 smithi164 ceph-mon[25711]: from='client.24159 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:23:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:47 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:47 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:23:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:47 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:47 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:47 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:23:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:47 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:47 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:23:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:47 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:47.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:47 smithi130 ceph-mon[23614]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-10T03:23:47.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:47 smithi130 ceph-mon[23614]: from='client.24159 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:23:47.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:47 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:47.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:47 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:23:47.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:47 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:47.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:47 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:47.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:47 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:23:47.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:47 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:47.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:47 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:23:47.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:47 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:23:48.314 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:48 smithi164 ceph-mon[25711]: Detected new or changed devices on smithi164 2024-09-10T03:23:48.314 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:48 smithi164 ceph-mon[25711]: Adjusting osd_memory_target on smithi164 to 15936M 2024-09-10T03:23:48.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:48 smithi130 ceph-mon[23614]: Detected new or changed devices on smithi164 2024-09-10T03:23:48.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:48 smithi130 ceph-mon[23614]: Adjusting osd_memory_target on smithi164 to 15936M 2024-09-10T03:23:49.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:49 smithi164 ceph-mon[25711]: pgmap v117: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-10T03:23:49.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:49 smithi164 ceph-mon[25711]: from='client.? 172.21.15.164:0/4225798169' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "780d0ca6-8b02-44dc-9aef-ed30b4e58dbc"}]: dispatch 2024-09-10T03:23:49.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:49 smithi164 ceph-mon[25711]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "780d0ca6-8b02-44dc-9aef-ed30b4e58dbc"}]: dispatch 2024-09-10T03:23:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:49 smithi164 ceph-mon[25711]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "780d0ca6-8b02-44dc-9aef-ed30b4e58dbc"}]': finished 2024-09-10T03:23:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:49 smithi164 ceph-mon[25711]: osdmap e37: 6 total, 5 up, 6 in 2024-09-10T03:23:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:49 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-10T03:23:49.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:49 smithi130 ceph-mon[23614]: pgmap v117: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-10T03:23:49.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:49 smithi130 ceph-mon[23614]: from='client.? 172.21.15.164:0/4225798169' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "780d0ca6-8b02-44dc-9aef-ed30b4e58dbc"}]: dispatch 2024-09-10T03:23:49.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:49 smithi130 ceph-mon[23614]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "780d0ca6-8b02-44dc-9aef-ed30b4e58dbc"}]: dispatch 2024-09-10T03:23:49.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:49 smithi130 ceph-mon[23614]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "780d0ca6-8b02-44dc-9aef-ed30b4e58dbc"}]': finished 2024-09-10T03:23:49.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:49 smithi130 ceph-mon[23614]: osdmap e37: 6 total, 5 up, 6 in 2024-09-10T03:23:49.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:49 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-10T03:23:50.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:50 smithi164 ceph-mon[25711]: from='client.? 172.21.15.164:0/2999690424' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-10T03:23:50.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:50 smithi130 ceph-mon[23614]: from='client.? 172.21.15.164:0/2999690424' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-10T03:23:51.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:51 smithi164 ceph-mon[25711]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-10T03:23:51.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:51 smithi130 ceph-mon[23614]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-10T03:23:53.366 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:53 smithi164 ceph-mon[25711]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-10T03:23:53.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:53 smithi130 ceph-mon[23614]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-10T03:23:55.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:55 smithi164 ceph-mon[25711]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-10T03:23:55.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:55 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-10T03:23:55.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:55 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:55.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:55 smithi130 ceph-mon[23614]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-10T03:23:55.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:55 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-10T03:23:55.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:55 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:23:56.329 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:56 smithi164 ceph-mon[25711]: Deploying daemon osd.5 on smithi164 2024-09-10T03:23:56.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:56 smithi130 ceph-mon[23614]: Deploying daemon osd.5 on smithi164 2024-09-10T03:23:57.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:57 smithi164 ceph-mon[25711]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-09-10T03:23:57.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:57 smithi130 ceph-mon[23614]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-09-10T03:23:59.348 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:23:59 smithi164 ceph-mon[25711]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-10T03:23:59.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:23:59 smithi130 ceph-mon[23614]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-10T03:24:01.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:00 smithi164 ceph-mon[25711]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-10T03:24:01.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:01.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:24:01.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:01.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:00 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:24:01.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:00 smithi130 ceph-mon[23614]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-10T03:24:01.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:00 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:01.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:00 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:24:01.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:00 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:01.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:00 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:24:02.624 INFO:teuthology.orchestra.run.smithi164.stdout:Created osd(s) 5 on host 'smithi164' 2024-09-10T03:24:03.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:02 smithi164 ceph-mon[25711]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-10T03:24:03.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:02 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:03.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:02 smithi130 ceph-mon[23614]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-10T03:24:03.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:02 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:03.580 DEBUG:teuthology.orchestra.run.smithi164:osd.5> sudo journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.5.service 2024-09-10T03:24:03.583 INFO:tasks.cephadm:Deploying osd.6 on smithi164 with /dev/vg_nvme/lv_2... 2024-09-10T03:24:03.583 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-10T03:24:04.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:03 smithi164 ceph-mon[25711]: from='osd.5 [v2:172.21.15.164:6808/3863596616,v1:172.21.15.164:6809/3863596616]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-10T03:24:04.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:03 smithi164 ceph-mon[25711]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-10T03:24:04.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:03 smithi130 ceph-mon[23614]: from='osd.5 [v2:172.21.15.164:6808/3863596616,v1:172.21.15.164:6809/3863596616]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-10T03:24:04.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:03 smithi130 ceph-mon[23614]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-10T03:24:05.066 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:04 smithi164 ceph-mon[25711]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-10T03:24:05.066 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:04 smithi164 ceph-mon[25711]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-10T03:24:05.066 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:04 smithi164 ceph-mon[25711]: osdmap e38: 6 total, 5 up, 6 in 2024-09-10T03:24:05.066 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:04 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-10T03:24:05.066 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:04 smithi164 ceph-mon[25711]: from='osd.5 [v2:172.21.15.164:6808/3863596616,v1:172.21.15.164:6809/3863596616]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-09-10T03:24:05.067 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:04 smithi164 ceph-mon[25711]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-09-10T03:24:05.067 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:04 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:05.067 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:24:04 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[32876]: 2024-09-10T03:24:04.928+0000 7fa747645700 -1 osd.5 0 waiting for initial osdmap 2024-09-10T03:24:05.067 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:24:04 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[32876]: 2024-09-10T03:24:04.934+0000 7fa7407d9700 -1 osd.5 39 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-10T03:24:05.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:04 smithi130 ceph-mon[23614]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-10T03:24:05.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:04 smithi130 ceph-mon[23614]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-10T03:24:05.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:04 smithi130 ceph-mon[23614]: osdmap e38: 6 total, 5 up, 6 in 2024-09-10T03:24:05.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:04 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-10T03:24:05.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:04 smithi130 ceph-mon[23614]: from='osd.5 [v2:172.21.15.164:6808/3863596616,v1:172.21.15.164:6809/3863596616]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-09-10T03:24:05.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:04 smithi130 ceph-mon[23614]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-09-10T03:24:05.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:04 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:05.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:05 smithi164 ceph-mon[25711]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2024-09-10T03:24:05.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:05 smithi164 ceph-mon[25711]: osdmap e39: 6 total, 5 up, 6 in 2024-09-10T03:24:05.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:05 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-10T03:24:05.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:05 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-10T03:24:05.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:05 smithi164 ceph-mon[25711]: osd.5 [v2:172.21.15.164:6808/3863596616,v1:172.21.15.164:6809/3863596616] boot 2024-09-10T03:24:05.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:05 smithi164 ceph-mon[25711]: osdmap e40: 6 total, 6 up, 6 in 2024-09-10T03:24:05.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:05 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-10T03:24:06.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:05 smithi130 ceph-mon[23614]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2024-09-10T03:24:06.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:05 smithi130 ceph-mon[23614]: osdmap e39: 6 total, 5 up, 6 in 2024-09-10T03:24:06.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:05 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-10T03:24:06.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:05 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-10T03:24:06.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:05 smithi130 ceph-mon[23614]: osd.5 [v2:172.21.15.164:6808/3863596616,v1:172.21.15.164:6809/3863596616] boot 2024-09-10T03:24:06.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:05 smithi130 ceph-mon[23614]: osdmap e40: 6 total, 6 up, 6 in 2024-09-10T03:24:06.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:05 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-10T03:24:07.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:06 smithi164 ceph-mon[25711]: purged_snaps scrub starts 2024-09-10T03:24:07.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:06 smithi164 ceph-mon[25711]: purged_snaps scrub ok 2024-09-10T03:24:07.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:06 smithi164 ceph-mon[25711]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-10T03:24:07.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:06 smithi164 ceph-mon[25711]: osdmap e41: 6 total, 6 up, 6 in 2024-09-10T03:24:07.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:06 smithi130 ceph-mon[23614]: purged_snaps scrub starts 2024-09-10T03:24:07.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:06 smithi130 ceph-mon[23614]: purged_snaps scrub ok 2024-09-10T03:24:07.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:06 smithi130 ceph-mon[23614]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-10T03:24:07.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:06 smithi130 ceph-mon[23614]: osdmap e41: 6 total, 6 up, 6 in 2024-09-10T03:24:07.626 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:24:07.649 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph orch daemon add osd smithi164:vg_nvme/lv_2 2024-09-10T03:24:08.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:08 smithi164 ceph-mon[25711]: osdmap e42: 6 total, 6 up, 6 in 2024-09-10T03:24:08.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:08 smithi130 ceph-mon[23614]: osdmap e42: 6 total, 6 up, 6 in 2024-09-10T03:24:09.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:09 smithi164 ceph-mon[25711]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:09.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:09 smithi164 ceph-mon[25711]: osdmap e43: 6 total, 6 up, 6 in 2024-09-10T03:24:09.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:09 smithi130 ceph-mon[23614]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:09.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:09 smithi130 ceph-mon[23614]: osdmap e43: 6 total, 6 up, 6 in 2024-09-10T03:24:10.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:10 smithi164 ceph-mon[25711]: osdmap e44: 6 total, 6 up, 6 in 2024-09-10T03:24:10.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:10 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:10.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:10 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:24:10.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:10 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:24:10.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:10 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:10.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:10 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:10.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:10 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:24:10.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:10 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:10.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:10 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:24:10.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:10 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:10.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:10 smithi130 ceph-mon[23614]: osdmap e44: 6 total, 6 up, 6 in 2024-09-10T03:24:10.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:10 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:10.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:10 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:24:10.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:10 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:24:10.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:10 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:10.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:10 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:10.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:10 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:24:10.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:10 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:10.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:10 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:24:10.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:10 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:11.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:11 smithi164 ceph-mon[25711]: Detected new or changed devices on smithi164 2024-09-10T03:24:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:11 smithi164 ceph-mon[25711]: Adjusting osd_memory_target on smithi164 to 7968M 2024-09-10T03:24:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:11 smithi164 ceph-mon[25711]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:11 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-10T03:24:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:11 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-10T03:24:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:11 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:11.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:11 smithi130 ceph-mon[23614]: Detected new or changed devices on smithi164 2024-09-10T03:24:11.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:11 smithi130 ceph-mon[23614]: Adjusting osd_memory_target on smithi164 to 7968M 2024-09-10T03:24:11.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:11 smithi130 ceph-mon[23614]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:11.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:11 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-10T03:24:11.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:11 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-10T03:24:11.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:11 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:12.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:12 smithi164 ceph-mon[25711]: from='client.24177 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:24:12.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:12 smithi130 ceph-mon[23614]: from='client.24177 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:24:13.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:13 smithi164 ceph-mon[25711]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:13.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:13 smithi164 ceph-mon[25711]: from='client.? 172.21.15.164:0/3596122846' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "998314ec-b60d-4c77-a341-aac116206398"}]: dispatch 2024-09-10T03:24:13.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:13 smithi164 ceph-mon[25711]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "998314ec-b60d-4c77-a341-aac116206398"}]: dispatch 2024-09-10T03:24:13.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:13 smithi164 ceph-mon[25711]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "998314ec-b60d-4c77-a341-aac116206398"}]': finished 2024-09-10T03:24:13.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:13 smithi164 ceph-mon[25711]: osdmap e45: 7 total, 6 up, 7 in 2024-09-10T03:24:13.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:13 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-10T03:24:13.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:13 smithi130 ceph-mon[23614]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:13.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:13 smithi130 ceph-mon[23614]: from='client.? 172.21.15.164:0/3596122846' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "998314ec-b60d-4c77-a341-aac116206398"}]: dispatch 2024-09-10T03:24:13.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:13 smithi130 ceph-mon[23614]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "998314ec-b60d-4c77-a341-aac116206398"}]: dispatch 2024-09-10T03:24:13.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:13 smithi130 ceph-mon[23614]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "998314ec-b60d-4c77-a341-aac116206398"}]': finished 2024-09-10T03:24:13.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:13 smithi130 ceph-mon[23614]: osdmap e45: 7 total, 6 up, 7 in 2024-09-10T03:24:13.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:13 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-10T03:24:14.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:14 smithi164 ceph-mon[25711]: from='client.? 172.21.15.164:0/2864818696' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-10T03:24:14.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:14 smithi130 ceph-mon[23614]: from='client.? 172.21.15.164:0/2864818696' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-10T03:24:15.345 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:15 smithi164 ceph-mon[25711]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:15.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:15 smithi130 ceph-mon[23614]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:17.233 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:17 smithi164 ceph-mon[25711]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:17.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:17 smithi130 ceph-mon[23614]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:19.375 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:19 smithi164 ceph-mon[25711]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:19.376 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:19 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-10T03:24:19.376 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:19 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:19.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:19 smithi130 ceph-mon[23614]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:19.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:19 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-10T03:24:19.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:19 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:20.277 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:20 smithi164 ceph-mon[25711]: Deploying daemon osd.6 on smithi164 2024-09-10T03:24:20.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:20 smithi130 ceph-mon[23614]: Deploying daemon osd.6 on smithi164 2024-09-10T03:24:21.451 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:21 smithi164 ceph-mon[25711]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:21.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:21 smithi130 ceph-mon[23614]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:23.365 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:23 smithi164 ceph-mon[25711]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:23.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:23 smithi130 ceph-mon[23614]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:24.866 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:24 smithi164 ceph-mon[25711]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:24.867 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:24 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:24.867 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:24 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:24:24.867 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:24 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:24.867 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:24 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:24:25.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:24 smithi130 ceph-mon[23614]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:25.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:24 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:25.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:24 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:24:25.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:24 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:25.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:24 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:24:27.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:27 smithi164 ceph-mon[25711]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:27.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:27 smithi164 ceph-mon[25711]: from='osd.6 [v2:172.21.15.164:6816/671812914,v1:172.21.15.164:6817/671812914]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-10T03:24:27.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:27 smithi164 ceph-mon[25711]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-10T03:24:27.512 INFO:teuthology.orchestra.run.smithi164.stdout:Created osd(s) 6 on host 'smithi164' 2024-09-10T03:24:27.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:27 smithi130 ceph-mon[23614]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:27.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:27 smithi130 ceph-mon[23614]: from='osd.6 [v2:172.21.15.164:6816/671812914,v1:172.21.15.164:6817/671812914]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-10T03:24:27.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:27 smithi130 ceph-mon[23614]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-10T03:24:28.086 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:28 smithi164 ceph-mon[25711]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-10T03:24:28.086 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:28 smithi164 ceph-mon[25711]: osdmap e46: 7 total, 6 up, 7 in 2024-09-10T03:24:28.086 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:28 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-10T03:24:28.086 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:28 smithi164 ceph-mon[25711]: from='osd.6 [v2:172.21.15.164:6816/671812914,v1:172.21.15.164:6817/671812914]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-09-10T03:24:28.086 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:28 smithi164 ceph-mon[25711]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-09-10T03:24:28.086 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:28 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:28.158 DEBUG:teuthology.orchestra.run.smithi164:osd.6> sudo journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.6.service 2024-09-10T03:24:28.160 INFO:tasks.cephadm:Deploying osd.7 on smithi164 with /dev/vg_nvme/lv_1... 2024-09-10T03:24:28.160 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-10T03:24:28.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:28 smithi130 ceph-mon[23614]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-10T03:24:28.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:28 smithi130 ceph-mon[23614]: osdmap e46: 7 total, 6 up, 7 in 2024-09-10T03:24:28.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:28 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-10T03:24:28.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:28 smithi130 ceph-mon[23614]: from='osd.6 [v2:172.21.15.164:6816/671812914,v1:172.21.15.164:6817/671812914]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-09-10T03:24:28.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:28 smithi130 ceph-mon[23614]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-09-10T03:24:28.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:28 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:29.343 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:29 smithi164 ceph-mon[25711]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:29.343 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:29 smithi164 ceph-mon[25711]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2024-09-10T03:24:29.343 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:29 smithi164 ceph-mon[25711]: osdmap e47: 7 total, 6 up, 7 in 2024-09-10T03:24:29.343 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:29 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-10T03:24:29.343 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:29 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-10T03:24:29.343 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:29 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:29.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:29 smithi130 ceph-mon[23614]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:29.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:29 smithi130 ceph-mon[23614]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2024-09-10T03:24:29.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:29 smithi130 ceph-mon[23614]: osdmap e47: 7 total, 6 up, 7 in 2024-09-10T03:24:29.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:29 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-10T03:24:29.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:29 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-10T03:24:29.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:29 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:30.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:30 smithi164 ceph-mon[25711]: purged_snaps scrub starts 2024-09-10T03:24:30.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:30 smithi164 ceph-mon[25711]: purged_snaps scrub ok 2024-09-10T03:24:30.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:30 smithi164 ceph-mon[25711]: osd.6 [v2:172.21.15.164:6816/671812914,v1:172.21.15.164:6817/671812914] boot 2024-09-10T03:24:30.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:30 smithi164 ceph-mon[25711]: osdmap e48: 7 total, 7 up, 7 in 2024-09-10T03:24:30.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:30 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-10T03:24:30.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:30 smithi164 ceph-mon[25711]: osdmap e49: 7 total, 7 up, 7 in 2024-09-10T03:24:30.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:30 smithi130 ceph-mon[23614]: purged_snaps scrub starts 2024-09-10T03:24:30.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:30 smithi130 ceph-mon[23614]: purged_snaps scrub ok 2024-09-10T03:24:30.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:30 smithi130 ceph-mon[23614]: osd.6 [v2:172.21.15.164:6816/671812914,v1:172.21.15.164:6817/671812914] boot 2024-09-10T03:24:30.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:30 smithi130 ceph-mon[23614]: osdmap e48: 7 total, 7 up, 7 in 2024-09-10T03:24:30.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:30 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-10T03:24:30.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:30 smithi130 ceph-mon[23614]: osdmap e49: 7 total, 7 up, 7 in 2024-09-10T03:24:31.429 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:31 smithi164 ceph-mon[25711]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:31.429 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:31 smithi164 ceph-mon[25711]: osdmap e50: 7 total, 7 up, 7 in 2024-09-10T03:24:31.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:31 smithi130 ceph-mon[23614]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-10T03:24:31.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:31 smithi130 ceph-mon[23614]: osdmap e50: 7 total, 7 up, 7 in 2024-09-10T03:24:31.734 INFO:teuthology.orchestra.run.smithi164.stdout: 2024-09-10T03:24:31.753 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph orch daemon add osd smithi164:vg_nvme/lv_1 2024-09-10T03:24:33.355 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:33 smithi164 ceph-mon[25711]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-10T03:24:33.356 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:33 smithi164 ceph-mon[25711]: osdmap e51: 7 total, 7 up, 7 in 2024-09-10T03:24:33.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:33 smithi130 ceph-mon[23614]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-10T03:24:33.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:33 smithi130 ceph-mon[23614]: osdmap e51: 7 total, 7 up, 7 in 2024-09-10T03:24:34.245 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:34 smithi164 ceph-mon[25711]: osdmap e52: 7 total, 7 up, 7 in 2024-09-10T03:24:34.245 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:34.245 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:24:34.245 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:24:34.245 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:24:34.245 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:34.245 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:34.245 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:24:34.245 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:34.246 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:24:34.246 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:34.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:34 smithi130 ceph-mon[23614]: osdmap e52: 7 total, 7 up, 7 in 2024-09-10T03:24:34.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:34.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:24:34.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:24:34.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:24:34.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:34.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:34.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:24:34.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:34.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:24:34.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:35.297 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:35 smithi164 ceph-mon[25711]: pgmap v156: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-10T03:24:35.298 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:35 smithi164 ceph-mon[25711]: Detected new or changed devices on smithi164 2024-09-10T03:24:35.298 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:35 smithi164 ceph-mon[25711]: Adjusting osd_memory_target on smithi164 to 5312M 2024-09-10T03:24:35.298 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-10T03:24:35.298 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-10T03:24:35.298 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:35.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:35 smithi130 ceph-mon[23614]: pgmap v156: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-10T03:24:35.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:35 smithi130 ceph-mon[23614]: Detected new or changed devices on smithi164 2024-09-10T03:24:35.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:35 smithi130 ceph-mon[23614]: Adjusting osd_memory_target on smithi164 to 5312M 2024-09-10T03:24:35.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-10T03:24:35.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-10T03:24:35.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:36.185 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:36 smithi164 ceph-mon[25711]: from='client.24195 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:24:36.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:36 smithi130 ceph-mon[23614]: from='client.24195 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:24:37.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:37 smithi164 ceph-mon[25711]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-09-10T03:24:37.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:37 smithi164 ceph-mon[25711]: from='client.? 172.21.15.164:0/3212511341' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2c8282f5-9513-496a-aa99-09ab791c8da6"}]: dispatch 2024-09-10T03:24:37.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:37 smithi164 ceph-mon[25711]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2c8282f5-9513-496a-aa99-09ab791c8da6"}]: dispatch 2024-09-10T03:24:37.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:37 smithi164 ceph-mon[25711]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2c8282f5-9513-496a-aa99-09ab791c8da6"}]': finished 2024-09-10T03:24:37.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:37 smithi164 ceph-mon[25711]: osdmap e53: 8 total, 7 up, 8 in 2024-09-10T03:24:37.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:37 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-10T03:24:37.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:37 smithi130 ceph-mon[23614]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-09-10T03:24:37.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:37 smithi130 ceph-mon[23614]: from='client.? 172.21.15.164:0/3212511341' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2c8282f5-9513-496a-aa99-09ab791c8da6"}]: dispatch 2024-09-10T03:24:37.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:37 smithi130 ceph-mon[23614]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2c8282f5-9513-496a-aa99-09ab791c8da6"}]: dispatch 2024-09-10T03:24:37.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:37 smithi130 ceph-mon[23614]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2c8282f5-9513-496a-aa99-09ab791c8da6"}]': finished 2024-09-10T03:24:37.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:37 smithi130 ceph-mon[23614]: osdmap e53: 8 total, 7 up, 8 in 2024-09-10T03:24:37.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:37 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-10T03:24:38.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:38 smithi164 ceph-mon[25711]: from='client.? 172.21.15.164:0/1275428152' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-10T03:24:38.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:38 smithi130 ceph-mon[23614]: from='client.? 172.21.15.164:0/1275428152' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-10T03:24:39.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:39 smithi164 ceph-mon[25711]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-10T03:24:39.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:39 smithi130 ceph-mon[23614]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-10T03:24:41.392 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:41 smithi164 ceph-mon[25711]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-09-10T03:24:41.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:41 smithi130 ceph-mon[23614]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-09-10T03:24:42.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:24:42.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:24:42.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:24:42.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:24:43.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:43 smithi164 ceph-mon[25711]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-10T03:24:43.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:43 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-10T03:24:43.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:43 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:43.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:43 smithi130 ceph-mon[23614]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-10T03:24:43.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:43 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-10T03:24:43.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:43 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:44.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:44 smithi164 ceph-mon[25711]: Deploying daemon osd.7 on smithi164 2024-09-10T03:24:44.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:44 smithi130 ceph-mon[23614]: Deploying daemon osd.7 on smithi164 2024-09-10T03:24:45.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:45 smithi164 ceph-mon[25711]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-10T03:24:45.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:45 smithi130 ceph-mon[23614]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-10T03:24:47.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:47 smithi164 ceph-mon[25711]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-10T03:24:47.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:47 smithi130 ceph-mon[23614]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-10T03:24:49.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:49 smithi164 ceph-mon[25711]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-10T03:24:49.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:49 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:49 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:24:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:49 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:49 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:24:49.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:49 smithi130 ceph-mon[23614]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-10T03:24:49.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:49 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:49.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:49 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:24:49.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:49 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:49.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:49 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:24:51.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:51 smithi164 ceph-mon[25711]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-10T03:24:51.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:51 smithi130 ceph-mon[23614]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-10T03:24:51.644 INFO:teuthology.orchestra.run.smithi164.stdout:Created osd(s) 7 on host 'smithi164' 2024-09-10T03:24:52.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:52 smithi164 ceph-mon[25711]: from='osd.7 [v2:172.21.15.164:6824/145508782,v1:172.21.15.164:6825/145508782]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-10T03:24:52.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:52 smithi164 ceph-mon[25711]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-10T03:24:52.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:52 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:52.263 DEBUG:teuthology.orchestra.run.smithi164:osd.7> sudo journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.7.service 2024-09-10T03:24:52.266 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-10T03:24:52.267 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph osd stat -f json 2024-09-10T03:24:52.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:52 smithi130 ceph-mon[23614]: from='osd.7 [v2:172.21.15.164:6824/145508782,v1:172.21.15.164:6825/145508782]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-10T03:24:52.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:52 smithi130 ceph-mon[23614]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-10T03:24:52.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:52 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:53.421 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:53 smithi164 ceph-mon[25711]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-10T03:24:53.421 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:53 smithi164 ceph-mon[25711]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-10T03:24:53.421 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:53 smithi164 ceph-mon[25711]: osdmap e54: 8 total, 7 up, 8 in 2024-09-10T03:24:53.421 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:53 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-10T03:24:53.421 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:53 smithi164 ceph-mon[25711]: from='osd.7 [v2:172.21.15.164:6824/145508782,v1:172.21.15.164:6825/145508782]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-09-10T03:24:53.421 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:53 smithi164 ceph-mon[25711]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-09-10T03:24:53.421 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:53 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:53.422 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:24:53 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[38889]: 2024-09-10T03:24:53.199+0000 7efc783e5700 -1 osd.7 0 waiting for initial osdmap 2024-09-10T03:24:53.422 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:24:53 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[38889]: 2024-09-10T03:24:53.207+0000 7efc72d7c700 -1 osd.7 55 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-10T03:24:53.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:53 smithi130 ceph-mon[23614]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-10T03:24:53.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:53 smithi130 ceph-mon[23614]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-10T03:24:53.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:53 smithi130 ceph-mon[23614]: osdmap e54: 8 total, 7 up, 8 in 2024-09-10T03:24:53.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:53 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-10T03:24:53.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:53 smithi130 ceph-mon[23614]: from='osd.7 [v2:172.21.15.164:6824/145508782,v1:172.21.15.164:6825/145508782]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-09-10T03:24:53.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:53 smithi130 ceph-mon[23614]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2024-09-10T03:24:53.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:53 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:54.261 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:24:54.310 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:54 smithi164 ceph-mon[25711]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2024-09-10T03:24:54.310 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:54 smithi164 ceph-mon[25711]: osdmap e55: 8 total, 7 up, 8 in 2024-09-10T03:24:54.310 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:54 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-10T03:24:54.310 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:54 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-10T03:24:54.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:54 smithi130 ceph-mon[23614]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2024-09-10T03:24:54.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:54 smithi130 ceph-mon[23614]: osdmap e55: 8 total, 7 up, 8 in 2024-09-10T03:24:54.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:54 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-10T03:24:54.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:54 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-10T03:24:54.966 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":56,"num_osds":8,"num_up_osds":8,"osd_up_since":1725938694,"num_in_osds":8,"osd_in_since":1725938676,"num_remapped_pgs":1} 2024-09-10T03:24:54.967 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph osd dump --format=json 2024-09-10T03:24:55.179 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:24:55.333 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:55 smithi164 ceph-mon[25711]: purged_snaps scrub starts 2024-09-10T03:24:55.333 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:55 smithi164 ceph-mon[25711]: purged_snaps scrub ok 2024-09-10T03:24:55.333 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:55 smithi164 ceph-mon[25711]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-10T03:24:55.333 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:55 smithi164 ceph-mon[25711]: osd.7 [v2:172.21.15.164:6824/145508782,v1:172.21.15.164:6825/145508782] boot 2024-09-10T03:24:55.333 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:55 smithi164 ceph-mon[25711]: osdmap e56: 8 total, 8 up, 8 in 2024-09-10T03:24:55.333 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:55 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-10T03:24:55.333 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:55 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/241131553' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-10T03:24:55.334 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:55 smithi164 ceph-mon[25711]: osdmap e57: 8 total, 8 up, 8 in 2024-09-10T03:24:55.480 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:55 smithi130 ceph-mon[23614]: purged_snaps scrub starts 2024-09-10T03:24:55.481 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:55 smithi130 ceph-mon[23614]: purged_snaps scrub ok 2024-09-10T03:24:55.481 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:55 smithi130 ceph-mon[23614]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-10T03:24:55.481 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:55 smithi130 ceph-mon[23614]: osd.7 [v2:172.21.15.164:6824/145508782,v1:172.21.15.164:6825/145508782] boot 2024-09-10T03:24:55.481 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:55 smithi130 ceph-mon[23614]: osdmap e56: 8 total, 8 up, 8 in 2024-09-10T03:24:55.481 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:55 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-10T03:24:55.481 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:55 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/241131553' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-10T03:24:55.481 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:55 smithi130 ceph-mon[23614]: osdmap e57: 8 total, 8 up, 8 in 2024-09-10T03:24:57.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:56 smithi130 ceph-mon[23614]: pgmap v172: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:24:57.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:56 smithi130 ceph-mon[23614]: Detected new or changed devices on smithi164 2024-09-10T03:24:57.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:57.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:24:57.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:24:57.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:24:57.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:24:57.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:56 smithi130 ceph-mon[23614]: Adjusting osd_memory_target on smithi164 to 3984M 2024-09-10T03:24:57.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:57.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:57.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:24:57.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:57.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:24:57.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:57.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:56 smithi130 ceph-mon[23614]: osdmap e58: 8 total, 8 up, 8 in 2024-09-10T03:24:57.106 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:24:57.106 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":59,"fsid":"5829498e-6f23-11ef-bcea-c7b262605968","created":"2024-09-10T03:18:50.375760+0000","modified":"2024-09-10T03:24:57.069012+0000","last_up_change":"2024-09-10T03:24:54.187826+0000","last_in_change":"2024-09-10T03:24:36.885356+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-10T03:22:55.855459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"28358d54-c6fe-475e-b7ac-60c43d222eae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1145235369},{"type":"v1","addr":"172.21.15.130:6803","nonce":1145235369}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1145235369},{"type":"v1","addr":"172.21.15.130:6805","nonce":1145235369}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":1145235369},{"type":"v1","addr":"172.21.15.130:6809","nonce":1145235369}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1145235369},{"type":"v1","addr":"172.21.15.130:6807","nonce":1145235369}]},"public_addr":"172.21.15.130:6803/1145235369","cluster_addr":"172.21.15.130:6805/1145235369","heartbeat_back_addr":"172.21.15.130:6809/1145235369","heartbeat_front_addr":"172.21.15.130:6807/1145235369","state":["exists","up"]},{"osd":1,"uuid":"c18f93f0-590b-4928-94b5-0559db3cb557","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":1791466340},{"type":"v1","addr":"172.21.15.130:6811","nonce":1791466340}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":1791466340},{"type":"v1","addr":"172.21.15.130:6813","nonce":1791466340}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1791466340},{"type":"v1","addr":"172.21.15.130:6817","nonce":1791466340}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":1791466340},{"type":"v1","addr":"172.21.15.130:6815","nonce":1791466340}]},"public_addr":"172.21.15.130:6811/1791466340","cluster_addr":"172.21.15.130:6813/1791466340","heartbeat_back_addr":"172.21.15.130:6817/1791466340","heartbeat_front_addr":"172.21.15.130:6815/1791466340","state":["exists","up"]},{"osd":2,"uuid":"78696ee3-3b0f-4c43-8f58-c9fd1a429c65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":2802507353},{"type":"v1","addr":"172.21.15.130:6819","nonce":2802507353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":2802507353},{"type":"v1","addr":"172.21.15.130:6821","nonce":2802507353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2802507353},{"type":"v1","addr":"172.21.15.130:6825","nonce":2802507353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":2802507353},{"type":"v1","addr":"172.21.15.130:6823","nonce":2802507353}]},"public_addr":"172.21.15.130:6819/2802507353","cluster_addr":"172.21.15.130:6821/2802507353","heartbeat_back_addr":"172.21.15.130:6825/2802507353","heartbeat_front_addr":"172.21.15.130:6823/2802507353","state":["exists","up"]},{"osd":3,"uuid":"1538e485-8857-450d-87f3-1ab837a97795","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":3722528714},{"type":"v1","addr":"172.21.15.130:6827","nonce":3722528714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":3722528714},{"type":"v1","addr":"172.21.15.130:6829","nonce":3722528714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3722528714},{"type":"v1","addr":"172.21.15.130:6833","nonce":3722528714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":3722528714},{"type":"v1","addr":"172.21.15.130:6831","nonce":3722528714}]},"public_addr":"172.21.15.130:6827/3722528714","cluster_addr":"172.21.15.130:6829/3722528714","heartbeat_back_addr":"172.21.15.130:6833/3722528714","heartbeat_front_addr":"172.21.15.130:6831/3722528714","state":["exists","up"]},{"osd":4,"uuid":"a381f70a-1f9a-4cab-9888-5e8f14e67528","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":1249950658},{"type":"v1","addr":"172.21.15.164:6801","nonce":1249950658}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":1249950658},{"type":"v1","addr":"172.21.15.164:6803","nonce":1249950658}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":1249950658},{"type":"v1","addr":"172.21.15.164:6807","nonce":1249950658}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":1249950658},{"type":"v1","addr":"172.21.15.164:6805","nonce":1249950658}]},"public_addr":"172.21.15.164:6801/1249950658","cluster_addr":"172.21.15.164:6803/1249950658","heartbeat_back_addr":"172.21.15.164:6807/1249950658","heartbeat_front_addr":"172.21.15.164:6805/1249950658","state":["exists","up"]},{"osd":5,"uuid":"780d0ca6-8b02-44dc-9aef-ed30b4e58dbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":3863596616},{"type":"v1","addr":"172.21.15.164:6809","nonce":3863596616}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":3863596616},{"type":"v1","addr":"172.21.15.164:6811","nonce":3863596616}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":3863596616},{"type":"v1","addr":"172.21.15.164:6815","nonce":3863596616}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":3863596616},{"type":"v1","addr":"172.21.15.164:6813","nonce":3863596616}]},"public_addr":"172.21.15.164:6809/3863596616","cluster_addr":"172.21.15.164:6811/3863596616","heartbeat_back_addr":"172.21.15.164:6815/3863596616","heartbeat_front_addr":"172.21.15.164:6813/3863596616","state":["exists","up"]},{"osd":6,"uuid":"998314ec-b60d-4c77-a341-aac116206398","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":671812914},{"type":"v1","addr":"172.21.15.164:6817","nonce":671812914}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":671812914},{"type":"v1","addr":"172.21.15.164:6819","nonce":671812914}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":671812914},{"type":"v1","addr":"172.21.15.164:6823","nonce":671812914}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":671812914},{"type":"v1","addr":"172.21.15.164:6821","nonce":671812914}]},"public_addr":"172.21.15.164:6817/671812914","cluster_addr":"172.21.15.164:6819/671812914","heartbeat_back_addr":"172.21.15.164:6823/671812914","heartbeat_front_addr":"172.21.15.164:6821/671812914","state":["exists","up"]},{"osd":7,"uuid":"2c8282f5-9513-496a-aa99-09ab791c8da6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6824","nonce":145508782},{"type":"v1","addr":"172.21.15.164:6825","nonce":145508782}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6826","nonce":145508782},{"type":"v1","addr":"172.21.15.164:6827","nonce":145508782}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6830","nonce":145508782},{"type":"v1","addr":"172.21.15.164:6831","nonce":145508782}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6828","nonce":145508782},{"type":"v1","addr":"172.21.15.164:6829","nonce":145508782}]},"public_addr":"172.21.15.164:6825/145508782","cluster_addr":"172.21.15.164:6827/145508782","heartbeat_back_addr":"172.21.15.164:6831/145508782","heartbeat_front_addr":"172.21.15.164:6829/145508782","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:21:52.563542+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:22:24.913343+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:22:51.695676+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:23:17.723973+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:23:40.239839+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:24:04.043415+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:24:27.748698+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:24:52.333430+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.130:0/3936575905":"2024-09-11T03:20:41.365610+0000","172.21.15.130:6801/921044500":"2024-09-11T03:20:41.365610+0000","172.21.15.130:0/1327285437":"2024-09-11T03:19:52.017921+0000","172.21.15.130:6800/921044500":"2024-09-11T03:20:41.365610+0000","172.21.15.130:6801/1510708754":"2024-09-11T03:19:52.017921+0000","172.21.15.130:0/4248168678":"2024-09-11T03:20:41.365610+0000","172.21.15.130:6800/1510708754":"2024-09-11T03:19:52.017921+0000","172.21.15.130:0/101528309":"2024-09-11T03:20:41.365610+0000","172.21.15.130:6800/2262033678":"2024-09-11T03:19:22.638737+0000","172.21.15.130:6801/2262033678":"2024-09-11T03:19:22.638737+0000","172.21.15.130:0/2889319894":"2024-09-11T03:19:52.017921+0000","172.21.15.130:0/2532600133":"2024-09-11T03:19:22.638737+0000","172.21.15.130:0/1907699823":"2024-09-11T03:19:52.017921+0000","172.21.15.130:0/2639128134":"2024-09-11T03:19:22.638737+0000","172.21.15.130:0/2846442362":"2024-09-11T03:19:22.638737+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-10T03:24:57.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:56 smithi164 ceph-mon[25711]: pgmap v172: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:24:57.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:56 smithi164 ceph-mon[25711]: Detected new or changed devices on smithi164 2024-09-10T03:24:57.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:57.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:24:57.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:24:57.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:24:57.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:24:57.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:56 smithi164 ceph-mon[25711]: Adjusting osd_memory_target on smithi164 to 3984M 2024-09-10T03:24:57.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:57.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:57.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:24:57.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:24:57.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:24:57.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:24:57.209 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:56 smithi164 ceph-mon[25711]: osdmap e58: 8 total, 8 up, 8 in 2024-09-10T03:24:57.838 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-10T03:22:55.855459+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-10T03:24:57.838 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-10T03:24:58.045 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:24:58.390 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:58 smithi130 ceph-mon[23614]: osdmap e59: 8 total, 8 up, 8 in 2024-09-10T03:24:58.390 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:58 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/925341982' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-10T03:24:58.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:58 smithi164 ceph-mon[25711]: osdmap e59: 8 total, 8 up, 8 in 2024-09-10T03:24:58.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:58 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/925341982' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-10T03:24:59.400 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:59 smithi130 ceph-mon[23614]: pgmap v175: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:24:59.400 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:24:59 smithi130 ceph-mon[23614]: osdmap e60: 8 total, 8 up, 8 in 2024-09-10T03:24:59.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:59 smithi164 ceph-mon[25711]: pgmap v175: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:24:59.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:24:59 smithi164 ceph-mon[25711]: osdmap e60: 8 total, 8 up, 8 in 2024-09-10T03:24:59.796 INFO:teuthology.orchestra.run.smithi130.stdout:pg_num: 1 2024-09-10T03:25:00.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:00 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/290188807' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-10T03:25:00.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:00 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/290188807' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-10T03:25:00.549 INFO:tasks.cephadm:Setting up client nodes... 2024-09-10T03:25:00.549 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-10T03:25:01.145 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:01 smithi130 ceph-mon[23614]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 114 KiB/s, 0 objects/s recovering 2024-09-10T03:25:01.358 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:01 smithi164 ceph-mon[25711]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 114 KiB/s, 0 objects/s recovering 2024-09-10T03:25:02.545 INFO:teuthology.orchestra.run.smithi130.stdout:[client.0] 2024-09-10T03:25:02.545 INFO:teuthology.orchestra.run.smithi130.stdout: key = AQAOvN9mgg5FIBAA6hGWcgIjlXC4MLZ6x+5bMA== 2024-09-10T03:25:03.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:03 smithi130 ceph-mon[23614]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-10T03:25:03.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:03 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3968530782' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-10T03:25:03.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:03 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3968530782' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-10T03:25:03.286 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-09-10T03:25:03.286 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-10T03:25:03.287 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-10T03:25:03.324 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph config log 1 --format=json 2024-09-10T03:25:03.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:03 smithi164 ceph-mon[25711]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-10T03:25:03.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:03 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/3968530782' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-10T03:25:03.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:03 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/3968530782' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-10T03:25:03.566 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:05.427 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:05 smithi130 ceph-mon[23614]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-09-10T03:25:05.428 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:25:05.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:05 smithi164 ceph-mon[25711]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-09-10T03:25:06.120 INFO:teuthology.orchestra.run.smithi130.stdout:[{"version":21,"timestamp":"2024-09-10T03:24:55.781226+0000","name":"","changes":[{"name":"osd/host:smithi164/osd_memory_target","previous_value":"5570077900","new_value":"4177558425"}]}] 2024-09-10T03:25:06.120 INFO:tasks.ceph_manager:config epoch is 21 2024-09-10T03:25:06.121 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-10T03:25:06.121 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-10T03:25:06.121 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph mgr dump --format=json 2024-09-10T03:25:06.329 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:06.450 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:06 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/1406029367' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-10T03:25:06.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:06 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/1406029367' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-10T03:25:07.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:07 smithi164 ceph-mon[25711]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-10T03:25:07.475 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:07 smithi130 ceph-mon[23614]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-10T03:25:08.174 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:25:08.924 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":19,"active_gid":14200,"active_name":"smithi130.axcmel","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":471014017},{"type":"v1","addr":"172.21.15.130:6801","nonce":471014017}]},"active_addr":"172.21.15.130:6801/471014017","active_change":"2024-09-10T03:20:41.365914+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14220,"name":"smithi164.bkfwap","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":998446174}]},{"addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":139534677}]},{"addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":2471498222}]},{"addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":3244525983}]}]}} 2024-09-10T03:25:08.931 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-10T03:25:08.931 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-10T03:25:08.932 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph osd dump --format=json 2024-09-10T03:25:09.140 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:09.260 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:09 smithi130 ceph-mon[23614]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-10T03:25:09.261 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:09 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3586035713' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-10T03:25:09.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:09 smithi164 ceph-mon[25711]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-10T03:25:09.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:09 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/3586035713' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-10T03:25:10.949 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:25:10.950 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":60,"fsid":"5829498e-6f23-11ef-bcea-c7b262605968","created":"2024-09-10T03:18:50.375760+0000","modified":"2024-09-10T03:24:58.072417+0000","last_up_change":"2024-09-10T03:24:54.187826+0000","last_in_change":"2024-09-10T03:24:36.885356+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-10T03:22:55.855459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"28358d54-c6fe-475e-b7ac-60c43d222eae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1145235369},{"type":"v1","addr":"172.21.15.130:6803","nonce":1145235369}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1145235369},{"type":"v1","addr":"172.21.15.130:6805","nonce":1145235369}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":1145235369},{"type":"v1","addr":"172.21.15.130:6809","nonce":1145235369}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1145235369},{"type":"v1","addr":"172.21.15.130:6807","nonce":1145235369}]},"public_addr":"172.21.15.130:6803/1145235369","cluster_addr":"172.21.15.130:6805/1145235369","heartbeat_back_addr":"172.21.15.130:6809/1145235369","heartbeat_front_addr":"172.21.15.130:6807/1145235369","state":["exists","up"]},{"osd":1,"uuid":"c18f93f0-590b-4928-94b5-0559db3cb557","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":1791466340},{"type":"v1","addr":"172.21.15.130:6811","nonce":1791466340}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":1791466340},{"type":"v1","addr":"172.21.15.130:6813","nonce":1791466340}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1791466340},{"type":"v1","addr":"172.21.15.130:6817","nonce":1791466340}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":1791466340},{"type":"v1","addr":"172.21.15.130:6815","nonce":1791466340}]},"public_addr":"172.21.15.130:6811/1791466340","cluster_addr":"172.21.15.130:6813/1791466340","heartbeat_back_addr":"172.21.15.130:6817/1791466340","heartbeat_front_addr":"172.21.15.130:6815/1791466340","state":["exists","up"]},{"osd":2,"uuid":"78696ee3-3b0f-4c43-8f58-c9fd1a429c65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":2802507353},{"type":"v1","addr":"172.21.15.130:6819","nonce":2802507353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":2802507353},{"type":"v1","addr":"172.21.15.130:6821","nonce":2802507353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2802507353},{"type":"v1","addr":"172.21.15.130:6825","nonce":2802507353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":2802507353},{"type":"v1","addr":"172.21.15.130:6823","nonce":2802507353}]},"public_addr":"172.21.15.130:6819/2802507353","cluster_addr":"172.21.15.130:6821/2802507353","heartbeat_back_addr":"172.21.15.130:6825/2802507353","heartbeat_front_addr":"172.21.15.130:6823/2802507353","state":["exists","up"]},{"osd":3,"uuid":"1538e485-8857-450d-87f3-1ab837a97795","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":3722528714},{"type":"v1","addr":"172.21.15.130:6827","nonce":3722528714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":3722528714},{"type":"v1","addr":"172.21.15.130:6829","nonce":3722528714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3722528714},{"type":"v1","addr":"172.21.15.130:6833","nonce":3722528714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":3722528714},{"type":"v1","addr":"172.21.15.130:6831","nonce":3722528714}]},"public_addr":"172.21.15.130:6827/3722528714","cluster_addr":"172.21.15.130:6829/3722528714","heartbeat_back_addr":"172.21.15.130:6833/3722528714","heartbeat_front_addr":"172.21.15.130:6831/3722528714","state":["exists","up"]},{"osd":4,"uuid":"a381f70a-1f9a-4cab-9888-5e8f14e67528","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":1249950658},{"type":"v1","addr":"172.21.15.164:6801","nonce":1249950658}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":1249950658},{"type":"v1","addr":"172.21.15.164:6803","nonce":1249950658}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":1249950658},{"type":"v1","addr":"172.21.15.164:6807","nonce":1249950658}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":1249950658},{"type":"v1","addr":"172.21.15.164:6805","nonce":1249950658}]},"public_addr":"172.21.15.164:6801/1249950658","cluster_addr":"172.21.15.164:6803/1249950658","heartbeat_back_addr":"172.21.15.164:6807/1249950658","heartbeat_front_addr":"172.21.15.164:6805/1249950658","state":["exists","up"]},{"osd":5,"uuid":"780d0ca6-8b02-44dc-9aef-ed30b4e58dbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":3863596616},{"type":"v1","addr":"172.21.15.164:6809","nonce":3863596616}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":3863596616},{"type":"v1","addr":"172.21.15.164:6811","nonce":3863596616}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":3863596616},{"type":"v1","addr":"172.21.15.164:6815","nonce":3863596616}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":3863596616},{"type":"v1","addr":"172.21.15.164:6813","nonce":3863596616}]},"public_addr":"172.21.15.164:6809/3863596616","cluster_addr":"172.21.15.164:6811/3863596616","heartbeat_back_addr":"172.21.15.164:6815/3863596616","heartbeat_front_addr":"172.21.15.164:6813/3863596616","state":["exists","up"]},{"osd":6,"uuid":"998314ec-b60d-4c77-a341-aac116206398","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":671812914},{"type":"v1","addr":"172.21.15.164:6817","nonce":671812914}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":671812914},{"type":"v1","addr":"172.21.15.164:6819","nonce":671812914}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":671812914},{"type":"v1","addr":"172.21.15.164:6823","nonce":671812914}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":671812914},{"type":"v1","addr":"172.21.15.164:6821","nonce":671812914}]},"public_addr":"172.21.15.164:6817/671812914","cluster_addr":"172.21.15.164:6819/671812914","heartbeat_back_addr":"172.21.15.164:6823/671812914","heartbeat_front_addr":"172.21.15.164:6821/671812914","state":["exists","up"]},{"osd":7,"uuid":"2c8282f5-9513-496a-aa99-09ab791c8da6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6824","nonce":145508782},{"type":"v1","addr":"172.21.15.164:6825","nonce":145508782}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6826","nonce":145508782},{"type":"v1","addr":"172.21.15.164:6827","nonce":145508782}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6830","nonce":145508782},{"type":"v1","addr":"172.21.15.164:6831","nonce":145508782}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6828","nonce":145508782},{"type":"v1","addr":"172.21.15.164:6829","nonce":145508782}]},"public_addr":"172.21.15.164:6825/145508782","cluster_addr":"172.21.15.164:6827/145508782","heartbeat_back_addr":"172.21.15.164:6831/145508782","heartbeat_front_addr":"172.21.15.164:6829/145508782","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:21:52.563542+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:22:24.913343+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:22:51.695676+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:23:17.723973+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:23:40.239839+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:24:04.043415+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:24:27.748698+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:24:52.333430+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.130:0/3936575905":"2024-09-11T03:20:41.365610+0000","172.21.15.130:6801/921044500":"2024-09-11T03:20:41.365610+0000","172.21.15.130:0/1327285437":"2024-09-11T03:19:52.017921+0000","172.21.15.130:6800/921044500":"2024-09-11T03:20:41.365610+0000","172.21.15.130:6801/1510708754":"2024-09-11T03:19:52.017921+0000","172.21.15.130:0/4248168678":"2024-09-11T03:20:41.365610+0000","172.21.15.130:6800/1510708754":"2024-09-11T03:19:52.017921+0000","172.21.15.130:0/101528309":"2024-09-11T03:20:41.365610+0000","172.21.15.130:6800/2262033678":"2024-09-11T03:19:22.638737+0000","172.21.15.130:6801/2262033678":"2024-09-11T03:19:22.638737+0000","172.21.15.130:0/2889319894":"2024-09-11T03:19:52.017921+0000","172.21.15.130:0/2532600133":"2024-09-11T03:19:22.638737+0000","172.21.15.130:0/1907699823":"2024-09-11T03:19:52.017921+0000","172.21.15.130:0/2639128134":"2024-09-11T03:19:22.638737+0000","172.21.15.130:0/2846442362":"2024-09-11T03:19:22.638737+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-10T03:25:11.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:11 smithi130 ceph-mon[23614]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s, 0 objects/s recovering 2024-09-10T03:25:11.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:11 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/134482923' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-10T03:25:11.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:11 smithi164 ceph-mon[25711]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s, 0 objects/s recovering 2024-09-10T03:25:11.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:11 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/134482923' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-10T03:25:11.636 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-10T03:25:11.636 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph osd dump --format=json 2024-09-10T03:25:11.841 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:13.147 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:13 smithi130 ceph-mon[23614]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:13.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:13 smithi164 ceph-mon[25711]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:13.554 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:25:13.554 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":60,"fsid":"5829498e-6f23-11ef-bcea-c7b262605968","created":"2024-09-10T03:18:50.375760+0000","modified":"2024-09-10T03:24:58.072417+0000","last_up_change":"2024-09-10T03:24:54.187826+0000","last_in_change":"2024-09-10T03:24:36.885356+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-10T03:22:55.855459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"28358d54-c6fe-475e-b7ac-60c43d222eae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1145235369},{"type":"v1","addr":"172.21.15.130:6803","nonce":1145235369}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1145235369},{"type":"v1","addr":"172.21.15.130:6805","nonce":1145235369}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":1145235369},{"type":"v1","addr":"172.21.15.130:6809","nonce":1145235369}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1145235369},{"type":"v1","addr":"172.21.15.130:6807","nonce":1145235369}]},"public_addr":"172.21.15.130:6803/1145235369","cluster_addr":"172.21.15.130:6805/1145235369","heartbeat_back_addr":"172.21.15.130:6809/1145235369","heartbeat_front_addr":"172.21.15.130:6807/1145235369","state":["exists","up"]},{"osd":1,"uuid":"c18f93f0-590b-4928-94b5-0559db3cb557","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":1791466340},{"type":"v1","addr":"172.21.15.130:6811","nonce":1791466340}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":1791466340},{"type":"v1","addr":"172.21.15.130:6813","nonce":1791466340}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1791466340},{"type":"v1","addr":"172.21.15.130:6817","nonce":1791466340}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":1791466340},{"type":"v1","addr":"172.21.15.130:6815","nonce":1791466340}]},"public_addr":"172.21.15.130:6811/1791466340","cluster_addr":"172.21.15.130:6813/1791466340","heartbeat_back_addr":"172.21.15.130:6817/1791466340","heartbeat_front_addr":"172.21.15.130:6815/1791466340","state":["exists","up"]},{"osd":2,"uuid":"78696ee3-3b0f-4c43-8f58-c9fd1a429c65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":2802507353},{"type":"v1","addr":"172.21.15.130:6819","nonce":2802507353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":2802507353},{"type":"v1","addr":"172.21.15.130:6821","nonce":2802507353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2802507353},{"type":"v1","addr":"172.21.15.130:6825","nonce":2802507353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":2802507353},{"type":"v1","addr":"172.21.15.130:6823","nonce":2802507353}]},"public_addr":"172.21.15.130:6819/2802507353","cluster_addr":"172.21.15.130:6821/2802507353","heartbeat_back_addr":"172.21.15.130:6825/2802507353","heartbeat_front_addr":"172.21.15.130:6823/2802507353","state":["exists","up"]},{"osd":3,"uuid":"1538e485-8857-450d-87f3-1ab837a97795","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":3722528714},{"type":"v1","addr":"172.21.15.130:6827","nonce":3722528714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":3722528714},{"type":"v1","addr":"172.21.15.130:6829","nonce":3722528714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3722528714},{"type":"v1","addr":"172.21.15.130:6833","nonce":3722528714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":3722528714},{"type":"v1","addr":"172.21.15.130:6831","nonce":3722528714}]},"public_addr":"172.21.15.130:6827/3722528714","cluster_addr":"172.21.15.130:6829/3722528714","heartbeat_back_addr":"172.21.15.130:6833/3722528714","heartbeat_front_addr":"172.21.15.130:6831/3722528714","state":["exists","up"]},{"osd":4,"uuid":"a381f70a-1f9a-4cab-9888-5e8f14e67528","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":1249950658},{"type":"v1","addr":"172.21.15.164:6801","nonce":1249950658}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":1249950658},{"type":"v1","addr":"172.21.15.164:6803","nonce":1249950658}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":1249950658},{"type":"v1","addr":"172.21.15.164:6807","nonce":1249950658}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":1249950658},{"type":"v1","addr":"172.21.15.164:6805","nonce":1249950658}]},"public_addr":"172.21.15.164:6801/1249950658","cluster_addr":"172.21.15.164:6803/1249950658","heartbeat_back_addr":"172.21.15.164:6807/1249950658","heartbeat_front_addr":"172.21.15.164:6805/1249950658","state":["exists","up"]},{"osd":5,"uuid":"780d0ca6-8b02-44dc-9aef-ed30b4e58dbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":3863596616},{"type":"v1","addr":"172.21.15.164:6809","nonce":3863596616}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":3863596616},{"type":"v1","addr":"172.21.15.164:6811","nonce":3863596616}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":3863596616},{"type":"v1","addr":"172.21.15.164:6815","nonce":3863596616}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":3863596616},{"type":"v1","addr":"172.21.15.164:6813","nonce":3863596616}]},"public_addr":"172.21.15.164:6809/3863596616","cluster_addr":"172.21.15.164:6811/3863596616","heartbeat_back_addr":"172.21.15.164:6815/3863596616","heartbeat_front_addr":"172.21.15.164:6813/3863596616","state":["exists","up"]},{"osd":6,"uuid":"998314ec-b60d-4c77-a341-aac116206398","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":671812914},{"type":"v1","addr":"172.21.15.164:6817","nonce":671812914}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":671812914},{"type":"v1","addr":"172.21.15.164:6819","nonce":671812914}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":671812914},{"type":"v1","addr":"172.21.15.164:6823","nonce":671812914}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":671812914},{"type":"v1","addr":"172.21.15.164:6821","nonce":671812914}]},"public_addr":"172.21.15.164:6817/671812914","cluster_addr":"172.21.15.164:6819/671812914","heartbeat_back_addr":"172.21.15.164:6823/671812914","heartbeat_front_addr":"172.21.15.164:6821/671812914","state":["exists","up"]},{"osd":7,"uuid":"2c8282f5-9513-496a-aa99-09ab791c8da6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6824","nonce":145508782},{"type":"v1","addr":"172.21.15.164:6825","nonce":145508782}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6826","nonce":145508782},{"type":"v1","addr":"172.21.15.164:6827","nonce":145508782}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6830","nonce":145508782},{"type":"v1","addr":"172.21.15.164:6831","nonce":145508782}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6828","nonce":145508782},{"type":"v1","addr":"172.21.15.164:6829","nonce":145508782}]},"public_addr":"172.21.15.164:6825/145508782","cluster_addr":"172.21.15.164:6827/145508782","heartbeat_back_addr":"172.21.15.164:6831/145508782","heartbeat_front_addr":"172.21.15.164:6829/145508782","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:21:52.563542+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:22:24.913343+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:22:51.695676+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:23:17.723973+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:23:40.239839+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:24:04.043415+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:24:27.748698+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-10T03:24:52.333430+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.130:0/3936575905":"2024-09-11T03:20:41.365610+0000","172.21.15.130:6801/921044500":"2024-09-11T03:20:41.365610+0000","172.21.15.130:0/1327285437":"2024-09-11T03:19:52.017921+0000","172.21.15.130:6800/921044500":"2024-09-11T03:20:41.365610+0000","172.21.15.130:6801/1510708754":"2024-09-11T03:19:52.017921+0000","172.21.15.130:0/4248168678":"2024-09-11T03:20:41.365610+0000","172.21.15.130:6800/1510708754":"2024-09-11T03:19:52.017921+0000","172.21.15.130:0/101528309":"2024-09-11T03:20:41.365610+0000","172.21.15.130:6800/2262033678":"2024-09-11T03:19:22.638737+0000","172.21.15.130:6801/2262033678":"2024-09-11T03:19:22.638737+0000","172.21.15.130:0/2889319894":"2024-09-11T03:19:52.017921+0000","172.21.15.130:0/2532600133":"2024-09-11T03:19:22.638737+0000","172.21.15.130:0/1907699823":"2024-09-11T03:19:52.017921+0000","172.21.15.130:0/2639128134":"2024-09-11T03:19:22.638737+0000","172.21.15.130:0/2846442362":"2024-09-11T03:19:22.638737+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-10T03:25:14.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:14 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/2692649195' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-10T03:25:14.331 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-10T03:25:14.331 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-10T03:25:14.332 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-10T03:25:14.332 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-10T03:25:14.333 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-10T03:25:14.333 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-10T03:25:14.334 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-10T03:25:14.334 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-10T03:25:14.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:14 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/2692649195' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-10T03:25:14.747 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:14.747 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:14.749 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:14.750 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:14.753 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:14.753 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:14.753 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:14.773 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:15.186 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:15 smithi130 ceph-mon[23614]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:15.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:15 smithi164 ceph-mon[25711]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:17.240 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:17 smithi130 ceph-mon[23614]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:17.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:17 smithi164 ceph-mon[25711]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:19.457 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:19 smithi130 ceph-mon[23614]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:19.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:19 smithi164 ceph-mon[25711]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:21.197 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:21 smithi130 ceph-mon[23614]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:21.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:21 smithi164 ceph-mon[25711]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:22.085 INFO:teuthology.orchestra.run.smithi130.stdout:240518168584 2024-09-10T03:25:22.086 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-10T03:25:23.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:23 smithi130 ceph-mon[23614]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:23.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:23 smithi164 ceph-mon[25711]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:25.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:25 smithi130 ceph-mon[23614]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:25.352 INFO:teuthology.orchestra.run.smithi130.stdout:81604378656 2024-09-10T03:25:25.352 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-10T03:25:25.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:25 smithi164 ceph-mon[25711]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:26.813 INFO:teuthology.orchestra.run.smithi130.stdout:115964117019 2024-09-10T03:25:26.814 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-10T03:25:27.349 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:27 smithi130 ceph-mon[23614]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:27.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:27 smithi164 ceph-mon[25711]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:28.447 INFO:teuthology.orchestra.run.smithi130.stdout:60129542183 2024-09-10T03:25:28.448 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-10T03:25:29.049 INFO:teuthology.orchestra.run.smithi130.stdout:206158430223 2024-09-10T03:25:29.049 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-10T03:25:29.301 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:29 smithi130 ceph-mon[23614]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:29.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:29 smithi164 ceph-mon[25711]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:29.836 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:29.837 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:29.840 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:29.841 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:31.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:31 smithi164 ceph-mon[25711]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:31.472 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:31 smithi130 ceph-mon[23614]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:31.783 INFO:teuthology.orchestra.run.smithi130.stdout:171798691859 2024-09-10T03:25:31.784 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-10T03:25:32.151 INFO:teuthology.orchestra.run.smithi130.stdout:137438953496 2024-09-10T03:25:32.152 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-10T03:25:33.376 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:33 smithi130 ceph-mon[23614]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:33.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:33 smithi164 ceph-mon[25711]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:35.393 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:35 smithi130 ceph-mon[23614]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:35.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:35 smithi164 ceph-mon[25711]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:35.806 INFO:teuthology.orchestra.run.smithi130.stdout:81604378658 2024-09-10T03:25:36.008 INFO:teuthology.orchestra.run.smithi130.stdout:38654705711 2024-09-10T03:25:36.008 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-10T03:25:36.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:36 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/2021003604' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-10T03:25:36.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:36 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/2021003604' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-10T03:25:36.928 INFO:teuthology.orchestra.run.smithi130.stdout:115964117023 2024-09-10T03:25:37.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:37 smithi130 ceph-mon[23614]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:37.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:37 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/2341576912' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-10T03:25:37.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:37 smithi164 ceph-mon[25711]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:37.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:37 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/2341576912' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-10T03:25:37.891 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117019 got 115964117023 for osd.3 2024-09-10T03:25:37.892 DEBUG:teuthology.parallel:result is None 2024-09-10T03:25:37.913 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:37.913 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:37.913 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:38.349 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:38.374 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378656 got 81604378658 for osd.2 2024-09-10T03:25:38.374 DEBUG:teuthology.parallel:result is None 2024-09-10T03:25:39.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:39 smithi130 ceph-mon[23614]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:39.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:39 smithi164 ceph-mon[25711]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:41.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:41 smithi164 ceph-mon[25711]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:41.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:41 smithi130 ceph-mon[23614]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:41.968 INFO:teuthology.orchestra.run.smithi130.stdout:60129542186 2024-09-10T03:25:42.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:25:42.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:25:42.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:42 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/2198634711' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-10T03:25:42.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:25:42.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:25:42.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:42 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/2198634711' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-10T03:25:43.434 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:43 smithi130 ceph-mon[23614]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:43.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:43 smithi164 ceph-mon[25711]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:43.635 INFO:teuthology.orchestra.run.smithi130.stdout:171798691862 2024-09-10T03:25:43.884 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542183 got 60129542186 for osd.1 2024-09-10T03:25:43.884 DEBUG:teuthology.parallel:result is None 2024-09-10T03:25:44.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:44 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/3820007593' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-10T03:25:44.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:44 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3820007593' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-10T03:25:45.049 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691859 got 171798691862 for osd.5 2024-09-10T03:25:45.049 DEBUG:teuthology.parallel:result is None 2024-09-10T03:25:45.271 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:45 smithi130 ceph-mon[23614]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:45.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:45 smithi164 ceph-mon[25711]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:46.094 INFO:teuthology.orchestra.run.smithi130.stdout:206158430226 2024-09-10T03:25:46.408 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:46 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/1397978715' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-10T03:25:46.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:46 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/1397978715' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-10T03:25:46.481 INFO:teuthology.orchestra.run.smithi130.stdout:38654705714 2024-09-10T03:25:47.124 INFO:teuthology.orchestra.run.smithi130.stdout:240518168589 2024-09-10T03:25:47.327 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:47 smithi130 ceph-mon[23614]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:47.327 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:47 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/1950183392' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-10T03:25:47.327 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:47 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/2538851440' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-10T03:25:47.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:47 smithi164 ceph-mon[25711]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:47 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/1950183392' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-10T03:25:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:47 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/2538851440' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-10T03:25:47.693 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705711 got 38654705714 for osd.0 2024-09-10T03:25:47.693 DEBUG:teuthology.parallel:result is None 2024-09-10T03:25:47.778 INFO:teuthology.orchestra.run.smithi130.stdout:137438953499 2024-09-10T03:25:48.373 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:48 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3842945523' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-10T03:25:48.396 INFO:tasks.cephadm.ceph_manager.ceph:need seq 137438953496 got 137438953499 for osd.4 2024-09-10T03:25:48.396 DEBUG:teuthology.parallel:result is None 2024-09-10T03:25:48.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:48 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/3842945523' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-10T03:25:48.792 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168584 got 240518168589 for osd.7 2024-09-10T03:25:48.792 DEBUG:teuthology.parallel:result is None 2024-09-10T03:25:49.280 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430223 got 206158430226 for osd.6 2024-09-10T03:25:49.281 DEBUG:teuthology.parallel:result is None 2024-09-10T03:25:49.281 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-10T03:25:49.281 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph pg dump --format=json 2024-09-10T03:25:49.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:49 smithi164 ceph-mon[25711]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:49.499 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:49.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:49 smithi130 ceph-mon[23614]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:51.256 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:25:51.258 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-09-10T03:25:51.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:51 smithi164 ceph-mon[25711]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:51.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:51 smithi130 ceph-mon[23614]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:51.934 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":202,"stamp":"2024-09-10T03:25:49.604093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48816,"kb_used_data":6384,"kb_used_omap":0,"kb_used_meta":42368,"kb_avail":749912400,"statfs":{"total":767960285184,"available":767910297600,"internally_reserved":0,"allocated":6537216,"data_stored":3572332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43384832},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002112"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-10T03:24:58.087753+0000","last_change":"2024-09-10T03:24:58.087753+0000","last_active":"2024-09-10T03:24:58.087753+0000","last_peered":"2024-09-10T03:24:58.087753+0000","last_clean":"2024-09-10T03:24:58.087753+0000","last_became_active":"2024-09-10T03:24:58.087310+0000","last_became_peered":"2024-09-10T03:24:58.087310+0000","last_unstale":"2024-09-10T03:24:58.087753+0000","last_undegraded":"2024-09-10T03:24:58.087753+0000","last_fullsized":"2024-09-10T03:24:58.087753+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-10T03:22:55.905681+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-10T03:22:55.905681+0000","last_clean_scrub_stamp":"2024-09-10T03:22:55.905681+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-11T10:31:06.466607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168590,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38300000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33800000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39900000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36899999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55300000000000005}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]}]},{"osd":6,"up_from":48,"seq":206158430227,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6184,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738968,"statfs":{"total":95995035648,"available":95988703232,"internally_reserved":0,"allocated":1146880,"data_stored":776930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 10 03:25:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.60099999999999998}]},{"osd":1,"last update":"Tue Sep 10 03:25:29 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.47499999999999998}]},{"osd":2,"last update":"Tue Sep 10 03:25:29 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.625},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.56899999999999995}]},{"osd":3,"last update":"Tue Sep 10 03:25:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.57899999999999996}]},{"osd":4,"last update":"Tue Sep 10 03:25:29 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.55600000000000005}]},{"osd":5,"last update":"Tue Sep 10 03:25:29 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.59199999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63900000000000001}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 10 03:25:31 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.34300000000000003,"15min":0.316},"min":{"1min":0.29999999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.34699999999999998,"15min":0.316},"min":{"1min":0.27400000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.58099999999999996,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.315}]},{"osd":2,"last update":"Tue Sep 10 03:25:01 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.41199999999999998,"15min":0.40600000000000003},"min":{"1min":0.26900000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.41599999999999998,"15min":0.40799999999999997},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.36799999999999999}]},{"osd":3,"last update":"Tue Sep 10 03:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46600000000000003,"15min":0.46700000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.33600000000000002}]},{"osd":4,"last update":"Tue Sep 10 03:25:43 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":1.1000000000000001,"15min":1.181},"min":{"1min":0.46000000000000002,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.82299999999999995,"5min":14.449,"15min":14.449},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.60199999999999998,"15min":0.59999999999999998},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.80100000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.52200000000000002}]},{"osd":5,"last update":"Tue Sep 10 03:25:07 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.56899999999999995}]},{"osd":6,"last update":"Tue Sep 10 03:25:31 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.49199999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.443}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Sep 10 03:25:32 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.32800000000000001,"15min":0.30199999999999999},"min":{"1min":0.222,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.52700000000000002,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.33900000000000002,"15min":0.317},"min":{"1min":0.26300000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.48399999999999999,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.315}]},{"osd":2,"last update":"Tue Sep 10 03:24:58 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.36599999999999999,"15min":0.35399999999999998},"min":{"1min":0.252,"5min":0.217,"15min":0.217},"max":{"1min":0.56799999999999995,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.39200000000000002,"15min":0.38300000000000001},"min":{"1min":0.28100000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.64400000000000002,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.38600000000000001}]},{"osd":3,"last update":"Tue Sep 10 03:25:24 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.45900000000000002,"15min":0.46500000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.623,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.436,"5min":0.47099999999999997,"15min":0.47699999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.59299999999999997,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.441}]},{"osd":4,"last update":"Tue Sep 10 03:24:46 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.55800000000000005}]},{"osd":5,"last update":"Tue Sep 10 03:25:05 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.76800000000000002}]},{"osd":6,"last update":"Tue Sep 10 03:25:30 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.69099999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73599999999999999}]}]},{"osd":2,"up_from":19,"seq":81604378661,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6244,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738908,"statfs":{"total":95995035648,"available":95988641792,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 10 03:24:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.36699999999999999,"15min":0.35799999999999998},"min":{"1min":0.247,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.38300000000000001,"15min":0.377},"min":{"1min":0.29799999999999999,"5min":0.248,"15min":0.248},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.52400000000000002}]},{"osd":1,"last update":"Tue Sep 10 03:24:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.38100000000000001,"15min":0.375},"min":{"1min":0.27600000000000002,"5min":0.254,"15min":0.254},"max":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.38900000000000001,"15min":0.379},"min":{"1min":0.33200000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.60599999999999998}]},{"osd":3,"last update":"Tue Sep 10 03:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.441,"15min":0.44},"min":{"1min":0.30199999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.441,"15min":0.44400000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.61499999999999999,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.57099999999999995}]},{"osd":4,"last update":"Tue Sep 10 03:24:47 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.79600000000000004}]},{"osd":5,"last update":"Tue Sep 10 03:25:05 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.752},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.77100000000000002}]},{"osd":6,"last update":"Tue Sep 10 03:25:30 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.78500000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71499999999999997}]}]},{"osd":3,"up_from":27,"seq":115964117025,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 10 03:25:19 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.435,"5min":0.41899999999999998,"15min":0.41699999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.36199999999999999}]},{"osd":1,"last update":"Tue Sep 10 03:25:19 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.46100000000000002,"15min":0.46500000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.435,"5min":0.437,"15min":0.437},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.63700000000000001,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.39900000000000002}]},{"osd":2,"last update":"Tue Sep 10 03:25:19 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.47599999999999998,"15min":0.47999999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.441,"5min":0.44900000000000001,"15min":0.45100000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.58799999999999997,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.38800000000000001}]},{"osd":4,"last update":"Tue Sep 10 03:25:44 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.58699999999999997,"15min":0.58299999999999996},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.58399999999999996,"15min":0.57799999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.89800000000000002,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.55400000000000005}]},{"osd":5,"last update":"Tue Sep 10 03:25:07 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.46100000000000002}]},{"osd":6,"last update":"Tue Sep 10 03:25:32 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.0680000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.61099999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51200000000000001}]}]},{"osd":4,"up_from":32,"seq":137438953500,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 10 03:25:44 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.55800000000000005,"15min":0.56399999999999995},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.68999999999999995,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.56899999999999995,"15min":0.57399999999999995},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.67600000000000005,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.39500000000000002}]},{"osd":1,"last update":"Tue Sep 10 03:25:44 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.54700000000000004,"15min":0.55400000000000005},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.63200000000000001,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.56999999999999995,"15min":0.57199999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.72799999999999998,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.41899999999999998}]},{"osd":2,"last update":"Tue Sep 10 03:25:44 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.69299999999999995,"5min":0.748,"15min":0.748},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.57499999999999996,"15min":0.57699999999999996},"min":{"1min":0.437,"5min":0.375,"15min":0.375},"max":{"1min":0.71599999999999997,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.45800000000000002}]},{"osd":3,"last update":"Tue Sep 10 03:25:44 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.58099999999999996,"15min":0.58599999999999997},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.72899999999999998,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.56699999999999995,"15min":0.57099999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.75,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.53300000000000003}]},{"osd":5,"last update":"Tue Sep 10 03:25:08 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.502}]},{"osd":6,"last update":"Tue Sep 10 03:25:29 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.61399999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66200000000000003}]}]},{"osd":5,"up_from":40,"seq":171798691863,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 10 03:25:10 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.51900000000000002}]},{"osd":1,"last update":"Tue Sep 10 03:25:10 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.58299999999999996}]},{"osd":2,"last update":"Tue Sep 10 03:25:10 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.624}]},{"osd":3,"last update":"Tue Sep 10 03:25:10 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.50700000000000001}]},{"osd":4,"last update":"Tue Sep 10 03:25:10 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.42699999999999999}]},{"osd":6,"last update":"Tue Sep 10 03:25:30 2024","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.44900000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59599999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-10T03:25:51.936 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph pg dump --format=json 2024-09-10T03:25:52.142 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:52.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:52 smithi164 ceph-mon[25711]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-10T03:25:52.471 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:52 smithi130 ceph-mon[23614]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-10T03:25:53.408 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:53 smithi130 ceph-mon[23614]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:53.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:53 smithi164 ceph-mon[25711]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:53.791 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:25:53.793 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-09-10T03:25:54.422 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":204,"stamp":"2024-09-10T03:25:53.604898+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48816,"kb_used_data":6384,"kb_used_omap":0,"kb_used_meta":42368,"kb_avail":749912400,"statfs":{"total":767960285184,"available":767910297600,"internally_reserved":0,"allocated":6537216,"data_stored":3572332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43384832},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002014"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-10T03:24:58.087753+0000","last_change":"2024-09-10T03:24:58.087753+0000","last_active":"2024-09-10T03:24:58.087753+0000","last_peered":"2024-09-10T03:24:58.087753+0000","last_clean":"2024-09-10T03:24:58.087753+0000","last_became_active":"2024-09-10T03:24:58.087310+0000","last_became_peered":"2024-09-10T03:24:58.087310+0000","last_unstale":"2024-09-10T03:24:58.087753+0000","last_undegraded":"2024-09-10T03:24:58.087753+0000","last_fullsized":"2024-09-10T03:24:58.087753+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-10T03:22:55.905681+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-10T03:22:55.905681+0000","last_clean_scrub_stamp":"2024-09-10T03:22:55.905681+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-11T10:31:06.466607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168590,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38300000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33800000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39900000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36899999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55300000000000005}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]}]},{"osd":6,"up_from":48,"seq":206158430228,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6184,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738968,"statfs":{"total":95995035648,"available":95988703232,"internally_reserved":0,"allocated":1146880,"data_stored":776930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 10 03:25:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.44600000000000001}]},{"osd":1,"last update":"Tue Sep 10 03:25:29 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.498}]},{"osd":2,"last update":"Tue Sep 10 03:25:29 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.48699999999999999}]},{"osd":3,"last update":"Tue Sep 10 03:25:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.46400000000000002}]},{"osd":4,"last update":"Tue Sep 10 03:25:29 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.51300000000000001}]},{"osd":5,"last update":"Tue Sep 10 03:25:29 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.43099999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57999999999999996}]}]},{"osd":1,"up_from":14,"seq":60129542189,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 10 03:25:31 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.34300000000000003,"15min":0.316},"min":{"1min":0.29999999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.34699999999999998,"15min":0.316},"min":{"1min":0.27400000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.58099999999999996,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.45800000000000002}]},{"osd":2,"last update":"Tue Sep 10 03:25:01 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.41199999999999998,"15min":0.40600000000000003},"min":{"1min":0.26900000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.41599999999999998,"15min":0.40799999999999997},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.438}]},{"osd":3,"last update":"Tue Sep 10 03:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46600000000000003,"15min":0.46700000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.53400000000000003}]},{"osd":4,"last update":"Tue Sep 10 03:25:43 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":1.1000000000000001,"15min":1.181},"min":{"1min":0.46000000000000002,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.82299999999999995,"5min":14.449,"15min":14.449},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.60199999999999998,"15min":0.59999999999999998},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.80100000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.72999999999999998}]},{"osd":5,"last update":"Tue Sep 10 03:25:07 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.77500000000000002}]},{"osd":6,"last update":"Tue Sep 10 03:25:31 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.72099999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78700000000000003}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Sep 10 03:25:32 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.32800000000000001,"15min":0.30199999999999999},"min":{"1min":0.222,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.52700000000000002,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.314},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.33900000000000002,"15min":0.317},"min":{"1min":0.26300000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.48399999999999999,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.32700000000000001}]},{"osd":2,"last update":"Tue Sep 10 03:24:58 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.36599999999999999,"15min":0.35399999999999998},"min":{"1min":0.252,"5min":0.217,"15min":0.217},"max":{"1min":0.56799999999999995,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.39200000000000002,"15min":0.38300000000000001},"min":{"1min":0.28100000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.64400000000000002,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.41699999999999998}]},{"osd":3,"last update":"Tue Sep 10 03:25:24 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.45900000000000002,"15min":0.46500000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.623,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.436,"5min":0.47099999999999997,"15min":0.47699999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.59299999999999997,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.72299999999999998}]},{"osd":4,"last update":"Tue Sep 10 03:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.57299999999999995,"15min":0.57399999999999995},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77500000000000002,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.60399999999999998,"15min":0.60799999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.625}]},{"osd":5,"last update":"Tue Sep 10 03:25:05 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.70099999999999996}]},{"osd":6,"last update":"Tue Sep 10 03:25:30 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.60699999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57899999999999996}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6244,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738908,"statfs":{"total":95995035648,"available":95988641792,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 10 03:24:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.36699999999999999,"15min":0.35799999999999998},"min":{"1min":0.247,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.38300000000000001,"15min":0.377},"min":{"1min":0.29799999999999999,"5min":0.248,"15min":0.248},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.315}]},{"osd":1,"last update":"Tue Sep 10 03:24:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.38100000000000001,"15min":0.375},"min":{"1min":0.27600000000000002,"5min":0.254,"15min":0.254},"max":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.38900000000000001,"15min":0.379},"min":{"1min":0.33200000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.30299999999999999}]},{"osd":3,"last update":"Tue Sep 10 03:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.441,"15min":0.44},"min":{"1min":0.30199999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.441,"15min":0.44400000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.61499999999999999,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.33600000000000002}]},{"osd":4,"last update":"Tue Sep 10 03:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.60499999999999998,"15min":0.59599999999999997},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.68700000000000006,"5min":0.61899999999999999,"15min":0.60799999999999998},"min":{"1min":0.46600000000000003,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.64100000000000001}]},{"osd":5,"last update":"Tue Sep 10 03:25:05 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.57399999999999995}]},{"osd":6,"last update":"Tue Sep 10 03:25:30 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.622},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.63}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65300000000000002}]}]},{"osd":3,"up_from":27,"seq":115964117026,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 10 03:25:19 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.378},{"interface":"front","average":{"1min":0.435,"5min":0.41899999999999998,"15min":0.41699999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.36599999999999999}]},{"osd":1,"last update":"Tue Sep 10 03:25:19 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.46100000000000002,"15min":0.46500000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.435,"5min":0.437,"15min":0.437},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.63700000000000001,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.41899999999999998}]},{"osd":2,"last update":"Tue Sep 10 03:25:19 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.47599999999999998,"15min":0.47999999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.437},{"interface":"front","average":{"1min":0.441,"5min":0.44900000000000001,"15min":0.45100000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.58799999999999997,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.39800000000000002}]},{"osd":4,"last update":"Tue Sep 10 03:25:44 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.58699999999999997,"15min":0.58299999999999996},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.58399999999999996,"15min":0.57799999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.89800000000000002,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.73599999999999999}]},{"osd":5,"last update":"Tue Sep 10 03:25:07 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.505},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.63400000000000001}]},{"osd":6,"last update":"Tue Sep 10 03:25:32 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.0680000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.64900000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50900000000000001}]}]},{"osd":4,"up_from":32,"seq":137438953501,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 10 03:25:44 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.55800000000000005,"15min":0.56399999999999995},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.68999999999999995,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.56899999999999995,"15min":0.57399999999999995},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.67600000000000005,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.33200000000000002}]},{"osd":1,"last update":"Tue Sep 10 03:25:44 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.54700000000000004,"15min":0.55400000000000005},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.63200000000000001,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.56999999999999995,"15min":0.57199999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.72799999999999998,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.377}]},{"osd":2,"last update":"Tue Sep 10 03:25:44 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.69299999999999995,"5min":0.748,"15min":0.748},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.57499999999999996,"15min":0.57699999999999996},"min":{"1min":0.437,"5min":0.375,"15min":0.375},"max":{"1min":0.71599999999999997,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.30299999999999999}]},{"osd":3,"last update":"Tue Sep 10 03:25:44 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.58099999999999996,"15min":0.58599999999999997},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.72899999999999998,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.56699999999999995,"15min":0.57099999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.75,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.41299999999999998}]},{"osd":5,"last update":"Tue Sep 10 03:25:08 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.49199999999999999}]},{"osd":6,"last update":"Tue Sep 10 03:25:29 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.53800000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.62}]}]},{"osd":5,"up_from":40,"seq":171798691864,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 10 03:25:10 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.46600000000000003}]},{"osd":1,"last update":"Tue Sep 10 03:25:10 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.41299999999999998}]},{"osd":2,"last update":"Tue Sep 10 03:25:10 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.379}]},{"osd":3,"last update":"Tue Sep 10 03:25:10 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.48899999999999999}]},{"osd":4,"last update":"Tue Sep 10 03:25:10 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.498},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.623}]},{"osd":6,"last update":"Tue Sep 10 03:25:30 2024","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.58299999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54300000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-10T03:25:54.423 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-10T03:25:54.423 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-10T03:25:54.423 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-10T03:25:54.424 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph health --format=json 2024-09-10T03:25:54.631 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/mon.smithi130/config 2024-09-10T03:25:55.394 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:55 smithi130 ceph-mon[23614]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:55.394 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:55 smithi130 ceph-mon[23614]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-10T03:25:55.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:55 smithi164 ceph-mon[25711]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:55.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:55 smithi164 ceph-mon[25711]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-10T03:25:56.206 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:25:56.206 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:25:56.206 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:56 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:25:56.417 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:25:56.417 INFO:teuthology.orchestra.run.smithi130.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-10T03:25:56.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:25:56.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:25:56.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:56 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:25:57.049 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-10T03:25:57.049 INFO:tasks.cephadm:Setup complete, yielding 2024-09-10T03:25:57.050 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-10T03:25:57.072 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi130.front.sepia.ceph.com 2024-09-10T03:25:57.072 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- bash -c 'ceph orch status' 2024-09-10T03:25:57.586 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:57 smithi130 ceph-mon[23614]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:57.587 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:57 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:25:57.587 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:57 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/1853795481' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-10T03:25:57.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:57 smithi164 ceph-mon[25711]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:57.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:57 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:25:57.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:57 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/1853795481' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-10T03:25:58.893 INFO:teuthology.orchestra.run.smithi130.stdout:Backend: cephadm 2024-09-10T03:25:58.893 INFO:teuthology.orchestra.run.smithi130.stdout:Available: Yes 2024-09-10T03:25:58.893 INFO:teuthology.orchestra.run.smithi130.stdout:Paused: No 2024-09-10T03:25:59.461 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:25:59 smithi130 ceph-mon[23614]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:25:59.462 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- bash -c 'ceph orch ps' 2024-09-10T03:25:59.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:25:59 smithi164 ceph-mon[25711]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:00.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:00 smithi164 ceph-mon[25711]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:00.734 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:00 smithi130 ceph-mon[23614]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:01.414 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:26:01.414 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (4m) 2m ago 5m 17.6M - ba2b418f427c a430c6fad8b8 2024-09-10T03:26:01.414 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (5m) 2m ago 5m 6400k - 17.2.0 e1d6a67b021e 138ed3222ba5 2024-09-10T03:26:01.414 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (5m) 68s ago 5m 6405k - 17.2.0 e1d6a67b021e 3623793da55d 2024-09-10T03:26:01.414 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (4m) 2m ago 5m 44.2M - 8.3.5 dad864ee21e9 1e4f86d07930 2024-09-10T03:26:01.414 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:9283 running (6m) 2m ago 6m 449M - 17.2.0 e1d6a67b021e 197f48917434 2024-09-10T03:26:01.414 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283 running (5m) 68s ago 5m 411M - 17.2.0 e1d6a67b021e 91b74e6868e2 2024-09-10T03:26:01.414 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (6m) 2m ago 7m 40.5M 2048M 17.2.0 e1d6a67b021e f2b698da8d6e 2024-09-10T03:26:01.415 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (5m) 68s ago 5m 39.1M 2048M 17.2.0 e1d6a67b021e 7a928b0b045c 2024-09-10T03:26:01.415 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (5m) 2m ago 5m 17.0M - 1dbe0e931976 fcb1f0cd878a 2024-09-10T03:26:01.415 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (4m) 68s ago 4m 17.4M - 1dbe0e931976 1b5c522b94af 2024-09-10T03:26:01.415 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (4m) 2m ago 4m 33.0M 3216M 17.2.0 e1d6a67b021e 1d66043abecd 2024-09-10T03:26:01.415 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (3m) 2m ago 3m 34.4M 3216M 17.2.0 e1d6a67b021e ca1ec04aecfa 2024-09-10T03:26:01.415 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (3m) 2m ago 3m 30.8M 3216M 17.2.0 e1d6a67b021e 5883f5d5e969 2024-09-10T03:26:01.415 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (2m) 2m ago 2m 13.1M 3216M 17.2.0 e1d6a67b021e b32084e1a404 2024-09-10T03:26:01.415 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (2m) 68s ago 2m 32.5M 3984M 17.2.0 e1d6a67b021e 3b5c9f00f64d 2024-09-10T03:26:01.415 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (2m) 68s ago 2m 34.0M 3984M 17.2.0 e1d6a67b021e f16357ac331e 2024-09-10T03:26:01.415 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (98s) 68s ago 97s 31.0M 3984M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:26:01.415 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (73s) 68s ago 73s 13.0M 3984M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:26:01.415 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (4m) 2m ago 4m 56.1M - 514e6a882f6e a405bca101aa 2024-09-10T03:26:01.608 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:01 smithi164 ceph-mon[25711]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:01.716 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:01 smithi130 ceph-mon[23614]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:02.041 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- bash -c 'ceph orch ls' 2024-09-10T03:26:03.618 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:03 smithi130 ceph-mon[23614]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:03.618 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:03 smithi130 ceph-mon[23614]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:03.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:03 smithi164 ceph-mon[25711]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:03.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:03 smithi164 ceph-mon[25711]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:03.909 INFO:teuthology.orchestra.run.smithi130.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-10T03:26:03.909 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-09-10T03:26:03.909 INFO:teuthology.orchestra.run.smithi130.stdout:crash 2/2 2m ago 6m * 2024-09-10T03:26:03.909 INFO:teuthology.orchestra.run.smithi130.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-09-10T03:26:03.910 INFO:teuthology.orchestra.run.smithi130.stdout:mgr 2/2 2m ago 6m count:2 2024-09-10T03:26:03.910 INFO:teuthology.orchestra.run.smithi130.stdout:mon 2/2 2m ago 5m smithi130:172.21.15.130=smithi130;smithi164:172.21.15.164=smithi164;count:2 2024-09-10T03:26:03.910 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-09-10T03:26:03.910 INFO:teuthology.orchestra.run.smithi130.stdout:osd 8 2m ago - 2024-09-10T03:26:03.910 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-09-10T03:26:04.548 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- bash -c 'ceph orch host ls' 2024-09-10T03:26:05.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:05 smithi164 ceph-mon[25711]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:05.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:05 smithi164 ceph-mon[25711]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:05.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:05 smithi130 ceph-mon[23614]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:05.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:05 smithi130 ceph-mon[23614]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:06.525 INFO:teuthology.orchestra.run.smithi130.stdout:HOST ADDR LABELS STATUS 2024-09-10T03:26:06.525 INFO:teuthology.orchestra.run.smithi130.stdout:smithi130 172.21.15.130 2024-09-10T03:26:06.525 INFO:teuthology.orchestra.run.smithi130.stdout:smithi164 172.21.15.164 2024-09-10T03:26:06.525 INFO:teuthology.orchestra.run.smithi130.stdout:2 hosts in cluster 2024-09-10T03:26:07.115 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- bash -c 'ceph orch device ls' 2024-09-10T03:26:07.654 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:07 smithi130 ceph-mon[23614]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:07.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:07 smithi164 ceph-mon[25711]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:08.617 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:08 smithi130 ceph-mon[23614]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:08.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:08 smithi164 ceph-mon[25711]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:09.144 INFO:teuthology.orchestra.run.smithi130.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-10T03:26:09.144 INFO:teuthology.orchestra.run.smithi130.stdout:smithi130 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620500Q1400BGN 400G LVM detected, locked 2024-09-10T03:26:09.144 INFO:teuthology.orchestra.run.smithi130.stdout:smithi164 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT623300H0400BGN 400G LVM detected, locked 2024-09-10T03:26:09.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:09 smithi164 ceph-mon[25711]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:09.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:09 smithi130 ceph-mon[23614]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:09.814 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-10T03:26:09.825 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi130.front.sepia.ceph.com 2024-09-10T03:26:09.825 DEBUG:teuthology.orchestra.run.smithi130:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-10T03:26:09.851 INFO:teuthology.orchestra.run.smithi130.stderr:+ systemctl stop nfs-server 2024-09-10T03:26:09.858 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi164.front.sepia.ceph.com 2024-09-10T03:26:09.859 DEBUG:teuthology.orchestra.run.smithi164:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-10T03:26:09.889 INFO:teuthology.orchestra.run.smithi164.stderr:+ systemctl stop nfs-server 2024-09-10T03:26:09.897 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-10T03:26:09.914 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi130.front.sepia.ceph.com 2024-09-10T03:26:09.914 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-09-10T03:26:10.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:10 smithi130 ceph-mon[23614]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:10.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:10 smithi164 ceph-mon[25711]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:11.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:11 smithi130 ceph-mon[23614]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:11.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:11 smithi164 ceph-mon[25711]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:12.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:12 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-10T03:26:12.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:12 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-10T03:26:13.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:13 smithi164 ceph-mon[25711]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:13.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:13 smithi164 ceph-mon[25711]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:13.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:13 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-10T03:26:13.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:13 smithi164 ceph-mon[25711]: osdmap e61: 8 total, 8 up, 8 in 2024-09-10T03:26:13.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:13 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-10T03:26:13.727 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:13 smithi130 ceph-mon[23614]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:13.727 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:13 smithi130 ceph-mon[23614]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:13.728 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:13 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-10T03:26:13.728 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:13 smithi130 ceph-mon[23614]: osdmap e61: 8 total, 8 up, 8 in 2024-09-10T03:26:13.728 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:13 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-10T03:26:13.728 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:13 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130[23610]: 2024-09-10T03:26:13.382+0000 7f8eea7ad700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-10T03:26:14.154 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-10T03:26:14.163 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-10T03:26:14.163 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph orch ls -f json 2024-09-10T03:26:14.401 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:14 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-10T03:26:14.401 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:14 smithi130 ceph-mon[23614]: osdmap e62: 8 total, 8 up, 8 in 2024-09-10T03:26:14.401 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:14 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-10T03:26:14.401 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:14 smithi130 ceph-mon[23614]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-10T03:26:14.402 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:14 smithi130 ceph-mon[23614]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-10T03:26:14.402 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:14 smithi130 ceph-mon[23614]: osdmap e63: 8 total, 8 up, 8 in 2024-09-10T03:26:14.402 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:14 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-10T03:26:14.402 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:14 smithi130 ceph-mon[23614]: fsmap foofs:0 2024-09-10T03:26:14.402 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:14 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:14.402 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:14 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:26:14.402 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:14 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:14.402 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:14 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:26:14.402 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:14 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:14.402 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:14 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:14.402 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:14 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi130.ukcxnb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-10T03:26:14.402 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:14 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi130.ukcxnb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-10T03:26:14.402 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:14 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:14.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:14 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-10T03:26:14.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:14 smithi164 ceph-mon[25711]: osdmap e62: 8 total, 8 up, 8 in 2024-09-10T03:26:14.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:14 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-10T03:26:14.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:14 smithi164 ceph-mon[25711]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-10T03:26:14.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:14 smithi164 ceph-mon[25711]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-10T03:26:14.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:14 smithi164 ceph-mon[25711]: osdmap e63: 8 total, 8 up, 8 in 2024-09-10T03:26:14.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:14 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-10T03:26:14.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:14 smithi164 ceph-mon[25711]: fsmap foofs:0 2024-09-10T03:26:14.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:14 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:14.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:14 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:26:14.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:14 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:14.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:14 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:26:14.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:14 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:14.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:14 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:14.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:14 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi130.ukcxnb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-10T03:26:14.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:14 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi130.ukcxnb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-10T03:26:14.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:14 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:15.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:15 smithi164 ceph-mon[25711]: Saving service mds.foofs spec with placement count:2 2024-09-10T03:26:15.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:15 smithi164 ceph-mon[25711]: Deploying daemon mds.foofs.smithi130.ukcxnb on smithi130 2024-09-10T03:26:15.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:15 smithi164 ceph-mon[25711]: pgmap v217: 65 pgs: 7 creating+peering, 1 active+clean, 57 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:15.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:15 smithi164 ceph-mon[25711]: osdmap e64: 8 total, 8 up, 8 in 2024-09-10T03:26:15.737 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:15 smithi130 ceph-mon[23614]: Saving service mds.foofs spec with placement count:2 2024-09-10T03:26:15.737 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:15 smithi130 ceph-mon[23614]: Deploying daemon mds.foofs.smithi130.ukcxnb on smithi130 2024-09-10T03:26:15.737 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:15 smithi130 ceph-mon[23614]: pgmap v217: 65 pgs: 7 creating+peering, 1 active+clean, 57 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:15.737 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:15 smithi130 ceph-mon[23614]: osdmap e64: 8 total, 8 up, 8 in 2024-09-10T03:26:16.608 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:16 smithi130 ceph-mon[23614]: osdmap e65: 8 total, 8 up, 8 in 2024-09-10T03:26:16.609 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:16 smithi130 ceph-mon[23614]: pgmap v220: 65 pgs: 30 active+clean, 5 creating+activating, 13 creating+peering, 17 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:16.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:16 smithi164 ceph-mon[25711]: osdmap e65: 8 total, 8 up, 8 in 2024-09-10T03:26:16.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:16 smithi164 ceph-mon[25711]: pgmap v220: 65 pgs: 30 active+clean, 5 creating+activating, 13 creating+peering, 17 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:17.377 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:26:17.377 INFO:teuthology.orchestra.run.smithi130.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-10T03:19:41.509427Z", "last_refresh": "2024-09-10T03:23:20.247049Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-10T03:19:35.290015Z", "last_refresh": "2024-09-10T03:23:20.247115Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-10T03:19:38.252027Z", "last_refresh": "2024-09-10T03:23:20.247178Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-10T03:26:13.400519Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-10T03:26:13.396434Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-10T03:19:33.242280Z", "last_refresh": "2024-09-10T03:23:20.246976Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi130:172.21.15.130=smithi130", "smithi164:172.21.15.164=smithi164"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-10T03:20:24.405687Z", "last_refresh": "2024-09-10T03:23:20.246834Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-10T03:19:40.299038Z", "last_refresh": "2024-09-10T03:23:20.247239Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-10T03:23:20.247363Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-10T03:19:37.032460Z", "last_refresh": "2024-09-10T03:23:20.247301Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-10T03:26:17.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:17 smithi164 ceph-mon[25711]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-10T03:26:17.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:17 smithi130 ceph-mon[23614]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-10T03:26:18.458 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-10T03:26:19.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:18 smithi130 ceph-mon[23614]: pgmap v221: 65 pgs: 40 active+clean, 5 creating+activating, 13 creating+peering, 7 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:19.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:18 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:19.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:18 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi164.awiezo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-10T03:26:19.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:18 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi164.awiezo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-10T03:26:19.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:18 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:19.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:18 smithi130 ceph-mon[23614]: Deploying daemon mds.foofs.smithi164.awiezo on smithi164 2024-09-10T03:26:19.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:18 smithi130 ceph-mon[23614]: mds.? [v2:172.21.15.130:6834/1167840712,v1:172.21.15.130:6835/1167840712] up:boot 2024-09-10T03:26:19.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:18 smithi130 ceph-mon[23614]: daemon mds.foofs.smithi130.ukcxnb assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-10T03:26:19.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:18 smithi130 ceph-mon[23614]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-10T03:26:19.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:18 smithi130 ceph-mon[23614]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-10T03:26:19.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:18 smithi130 ceph-mon[23614]: Cluster is now healthy 2024-09-10T03:26:19.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:18 smithi130 ceph-mon[23614]: fsmap foofs:0 1 up:standby 2024-09-10T03:26:19.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:18 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi130.ukcxnb"}]: dispatch 2024-09-10T03:26:19.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:18 smithi130 ceph-mon[23614]: fsmap foofs:1 {0=foofs.smithi130.ukcxnb=up:creating} 2024-09-10T03:26:19.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:18 smithi130 ceph-mon[23614]: daemon mds.foofs.smithi130.ukcxnb is now active in filesystem foofs as rank 0 2024-09-10T03:26:19.390 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:18 smithi164 ceph-mon[25711]: pgmap v221: 65 pgs: 40 active+clean, 5 creating+activating, 13 creating+peering, 7 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:19.390 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:18 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:19.390 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:18 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi164.awiezo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-10T03:26:19.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:18 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi164.awiezo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-10T03:26:19.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:18 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:19.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:18 smithi164 ceph-mon[25711]: Deploying daemon mds.foofs.smithi164.awiezo on smithi164 2024-09-10T03:26:19.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:18 smithi164 ceph-mon[25711]: mds.? [v2:172.21.15.130:6834/1167840712,v1:172.21.15.130:6835/1167840712] up:boot 2024-09-10T03:26:19.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:18 smithi164 ceph-mon[25711]: daemon mds.foofs.smithi130.ukcxnb assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-10T03:26:19.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:18 smithi164 ceph-mon[25711]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-10T03:26:19.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:18 smithi164 ceph-mon[25711]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-10T03:26:19.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:18 smithi164 ceph-mon[25711]: Cluster is now healthy 2024-09-10T03:26:19.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:18 smithi164 ceph-mon[25711]: fsmap foofs:0 1 up:standby 2024-09-10T03:26:19.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:18 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi130.ukcxnb"}]: dispatch 2024-09-10T03:26:19.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:18 smithi164 ceph-mon[25711]: fsmap foofs:1 {0=foofs.smithi130.ukcxnb=up:creating} 2024-09-10T03:26:19.391 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:18 smithi164 ceph-mon[25711]: daemon mds.foofs.smithi130.ukcxnb is now active in filesystem foofs as rank 0 2024-09-10T03:26:19.460 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph orch ls -f json 2024-09-10T03:26:20.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:20 smithi164 ceph-mon[25711]: mds.? [v2:172.21.15.130:6834/1167840712,v1:172.21.15.130:6835/1167840712] up:active 2024-09-10T03:26:20.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:20 smithi164 ceph-mon[25711]: fsmap foofs:1 {0=foofs.smithi130.ukcxnb=up:active} 2024-09-10T03:26:20.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:20 smithi164 ceph-mon[25711]: pgmap v222: 65 pgs: 60 active+clean, 5 creating+activating; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2024-09-10T03:26:20.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:20 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:20.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:20 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:26:20.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:20 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:20.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:20 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:26:20.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:20 smithi130 ceph-mon[23614]: mds.? [v2:172.21.15.130:6834/1167840712,v1:172.21.15.130:6835/1167840712] up:active 2024-09-10T03:26:20.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:20 smithi130 ceph-mon[23614]: fsmap foofs:1 {0=foofs.smithi130.ukcxnb=up:active} 2024-09-10T03:26:20.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:20 smithi130 ceph-mon[23614]: pgmap v222: 65 pgs: 60 active+clean, 5 creating+activating; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2024-09-10T03:26:20.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:20 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:20.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:20 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:26:20.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:20 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:20.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:20 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:26:21.604 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:26:21.604 INFO:teuthology.orchestra.run.smithi130.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-10T03:19:41.509427Z", "last_refresh": "2024-09-10T03:23:20.247049Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-10T03:19:35.290015Z", "last_refresh": "2024-09-10T03:23:20.247115Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-10T03:19:38.252027Z", "last_refresh": "2024-09-10T03:23:20.247178Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-10T03:26:13.400519Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-10T03:26:13.396434Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-10T03:19:33.242280Z", "last_refresh": "2024-09-10T03:23:20.246976Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi130:172.21.15.130=smithi130", "smithi164:172.21.15.164=smithi164"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-10T03:20:24.405687Z", "last_refresh": "2024-09-10T03:23:20.246834Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-10T03:19:40.299038Z", "last_refresh": "2024-09-10T03:23:20.247239Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-10T03:23:20.247363Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-10T03:19:37.032460Z", "last_refresh": "2024-09-10T03:23:20.247301Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-10T03:26:22.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:22 smithi164 ceph-mon[25711]: mds.? [v2:172.21.15.164:6832/3151002352,v1:172.21.15.164:6833/3151002352] up:boot 2024-09-10T03:26:22.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:22 smithi164 ceph-mon[25711]: fsmap foofs:1 {0=foofs.smithi130.ukcxnb=up:active} 1 up:standby 2024-09-10T03:26:22.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:22 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi164.awiezo"}]: dispatch 2024-09-10T03:26:22.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:22 smithi164 ceph-mon[25711]: fsmap foofs:1 {0=foofs.smithi130.ukcxnb=up:active} 1 up:standby 2024-09-10T03:26:22.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:22 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:22.489 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:22 smithi130 ceph-mon[23614]: mds.? [v2:172.21.15.164:6832/3151002352,v1:172.21.15.164:6833/3151002352] up:boot 2024-09-10T03:26:22.489 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:22 smithi130 ceph-mon[23614]: fsmap foofs:1 {0=foofs.smithi130.ukcxnb=up:active} 1 up:standby 2024-09-10T03:26:22.489 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:22 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi164.awiezo"}]: dispatch 2024-09-10T03:26:22.489 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:22 smithi130 ceph-mon[23614]: fsmap foofs:1 {0=foofs.smithi130.ukcxnb=up:active} 1 up:standby 2024-09-10T03:26:22.489 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:22 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:22.515 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-10T03:26:23.376 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:23 smithi130 ceph-mon[23614]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-10T03:26:23.376 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:23 smithi130 ceph-mon[23614]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-10T03:26:23.376 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:23 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:23.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:23 smithi164 ceph-mon[25711]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-10T03:26:23.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:23 smithi164 ceph-mon[25711]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-10T03:26:23.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:23 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:23.516 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph orch ls -f json 2024-09-10T03:26:25.445 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:25 smithi130 ceph-mon[23614]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-10T03:26:25.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:25 smithi164 ceph-mon[25711]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-10T03:26:26.472 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:26:26.472 INFO:teuthology.orchestra.run.smithi130.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-10T03:19:41.509427Z", "last_refresh": "2024-09-10T03:23:20.247049Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-10T03:19:35.290015Z", "last_refresh": "2024-09-10T03:23:20.247115Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-10T03:19:38.252027Z", "last_refresh": "2024-09-10T03:23:20.247178Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-10T03:26:13.400519Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-10T03:26:13.396434Z", "last_refresh": "2024-09-10T03:26:22.466364Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-10T03:19:33.242280Z", "last_refresh": "2024-09-10T03:23:20.246976Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi130:172.21.15.130=smithi130", "smithi164:172.21.15.164=smithi164"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-10T03:20:24.405687Z", "last_refresh": "2024-09-10T03:23:20.246834Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-10T03:19:40.299038Z", "last_refresh": "2024-09-10T03:23:20.247239Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-10T03:23:20.247363Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-10T03:19:37.032460Z", "last_refresh": "2024-09-10T03:23:20.247301Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-10T03:26:27.282 INFO:tasks.cephadm:mds.foofs has 1/2 2024-09-10T03:26:27.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:27 smithi164 ceph-mon[25711]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-10T03:26:27.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:27 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:27.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:27 smithi130 ceph-mon[23614]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-10T03:26:27.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:27 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:28.283 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph orch ls -f json 2024-09-10T03:26:28.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:28 smithi164 ceph-mon[25711]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-10T03:26:28.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:28 smithi130 ceph-mon[23614]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-10T03:26:29.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:29 smithi164 ceph-mon[25711]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-10T03:26:29.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:29 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:29.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:29 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:29.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:29 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:26:29.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:29 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:29.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:29 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:26:29.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:29 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:29.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:29 smithi130 ceph-mon[23614]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-10T03:26:29.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:29 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:29.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:29 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:29.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:29 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:26:29.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:29 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:29.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:29 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:26:29.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:29 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:30.234 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:26:30.234 INFO:teuthology.orchestra.run.smithi130.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-10T03:19:41.509427Z", "last_refresh": "2024-09-10T03:26:28.172385Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-10T03:19:35.290015Z", "last_refresh": "2024-09-10T03:26:22.465674Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-10T03:19:38.252027Z", "last_refresh": "2024-09-10T03:26:28.172510Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-10T03:26:13.400519Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-10T03:26:13.396434Z", "last_refresh": "2024-09-10T03:26:22.466364Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-10T03:19:33.242280Z", "last_refresh": "2024-09-10T03:26:22.465858Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi130:172.21.15.130=smithi130", "smithi164:172.21.15.164=smithi164"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-10T03:20:24.405687Z", "last_refresh": "2024-09-10T03:26:22.465941Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-10T03:19:40.299038Z", "last_refresh": "2024-09-10T03:26:22.466030Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-10T03:26:22.466097Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-10T03:19:37.032460Z", "last_refresh": "2024-09-10T03:26:28.172631Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-10T03:26:30.883 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-10T03:26:30.883 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-10T03:26:30.892 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi130.front.sepia.ceph.com 2024-09-10T03:26:30.892 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-10T03:26:31.453 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:31 smithi130 ceph-mon[23614]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-10T03:26:31.453 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:31 smithi130 ceph-mon[23614]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-10T03:26:31.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:31 smithi164 ceph-mon[25711]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-10T03:26:31.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:31 smithi164 ceph-mon[25711]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-10T03:26:33.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:33 smithi164 ceph-mon[25711]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-10T03:26:33.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:33 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-10T03:26:33.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:33 smithi130 ceph-mon[23614]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-10T03:26:33.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:33 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-10T03:26:34.221 INFO:teuthology.orchestra.run.smithi130.stdout:NFS Cluster Created Successfully 2024-09-10T03:26:34.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:34 smithi164 ceph-mon[25711]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:34.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-10T03:26:34.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:34 smithi164 ceph-mon[25711]: osdmap e66: 8 total, 8 up, 8 in 2024-09-10T03:26:34.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:34 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-10T03:26:34.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:34 smithi130 ceph-mon[23614]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:34.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-10T03:26:34.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:34 smithi130 ceph-mon[23614]: osdmap e66: 8 total, 8 up, 8 in 2024-09-10T03:26:34.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:34 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-10T03:26:34.983 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-10T03:26:35.234 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: pgmap v230: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:35.234 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-10T03:26:35.234 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: osdmap e67: 8 total, 8 up, 8 in 2024-09-10T03:26:35.234 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: Saving service nfs.foo spec with placement count:2 2024-09-10T03:26:35.234 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:35.235 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:26:35.235 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:35.235 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:26:35.235 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:35.235 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:35.235 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: Creating key for client.nfs.foo.0.0.smithi130.sanuoy 2024-09-10T03:26:35.235 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi130.sanuoy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-10T03:26:35.235 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi130.sanuoy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-10T03:26:35.235 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-10T03:26:35.235 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-10T03:26:35.235 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-10T03:26:35.235 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:35.236 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-10T03:26:35.236 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-10T03:26:35.236 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi130.sanuoy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-10T03:26:35.236 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi130.sanuoy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-10T03:26:35.236 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:35 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:35.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: pgmap v230: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-10T03:26:35.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-10T03:26:35.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: osdmap e67: 8 total, 8 up, 8 in 2024-09-10T03:26:35.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: Saving service nfs.foo spec with placement count:2 2024-09-10T03:26:35.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:35.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:26:35.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:35.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:26:35.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:35.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:35.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: Creating key for client.nfs.foo.0.0.smithi130.sanuoy 2024-09-10T03:26:35.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi130.sanuoy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-10T03:26:35.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi130.sanuoy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-10T03:26:35.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-10T03:26:35.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-10T03:26:35.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-10T03:26:35.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:35.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-10T03:26:35.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-10T03:26:35.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi130.sanuoy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-10T03:26:35.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi130.sanuoy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-10T03:26:35.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:35 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:36.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:36 smithi130 ceph-mon[23614]: Rados config object exists: conf-nfs.foo 2024-09-10T03:26:36.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:36 smithi130 ceph-mon[23614]: Creating key for client.nfs.foo.0.0.smithi130.sanuoy-rgw 2024-09-10T03:26:36.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:36 smithi130 ceph-mon[23614]: Deploying daemon nfs.foo.0.0.smithi130.sanuoy on smithi130 2024-09-10T03:26:36.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:36 smithi130 ceph-mon[23614]: osdmap e68: 8 total, 8 up, 8 in 2024-09-10T03:26:36.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:36 smithi164 ceph-mon[25711]: Rados config object exists: conf-nfs.foo 2024-09-10T03:26:36.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:36 smithi164 ceph-mon[25711]: Creating key for client.nfs.foo.0.0.smithi130.sanuoy-rgw 2024-09-10T03:26:36.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:36 smithi164 ceph-mon[25711]: Deploying daemon nfs.foo.0.0.smithi130.sanuoy on smithi130 2024-09-10T03:26:36.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:36 smithi164 ceph-mon[25711]: osdmap e68: 8 total, 8 up, 8 in 2024-09-10T03:26:37.485 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:37 smithi130 ceph-mon[23614]: pgmap v233: 97 pgs: 16 unknown, 81 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-10T03:26:37.485 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:37 smithi130 ceph-mon[23614]: mgrmap e20: smithi130.axcmel(active, since 5m), standbys: smithi164.bkfwap 2024-09-10T03:26:37.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:37 smithi164 ceph-mon[25711]: pgmap v233: 97 pgs: 16 unknown, 81 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-10T03:26:37.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:37 smithi164 ceph-mon[25711]: mgrmap e20: smithi130.axcmel(active, since 5m), standbys: smithi164.bkfwap 2024-09-10T03:26:38.180 INFO:teuthology.orchestra.run.smithi130.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-10T03:26:38.180 INFO:teuthology.orchestra.run.smithi130.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-10T03:26:38.180 INFO:teuthology.orchestra.run.smithi130.stderr:Error EINVAL: invalid command 2024-09-10T03:26:38.548 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:26:38.548 INFO:teuthology.orchestra.run.smithi130.stdout: "bind": "/fake", 2024-09-10T03:26:38.548 INFO:teuthology.orchestra.run.smithi130.stdout: "fs": "foofs", 2024-09-10T03:26:38.548 INFO:teuthology.orchestra.run.smithi130.stdout: "path": "/", 2024-09-10T03:26:38.548 INFO:teuthology.orchestra.run.smithi130.stdout: "cluster": "foo", 2024-09-10T03:26:38.548 INFO:teuthology.orchestra.run.smithi130.stdout: "mode": "RW" 2024-09-10T03:26:38.549 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:26:39.179 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-10T03:26:39.479 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:39 smithi130 ceph-mon[23614]: pgmap v234: 97 pgs: 14 unknown, 83 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-10T03:26:39.479 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:39 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-10T03:26:39.479 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:39 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-10T03:26:39.479 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:39 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:39.479 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:39 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.wfpoty", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-10T03:26:39.479 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:39 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.wfpoty", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-10T03:26:39.479 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:39 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-10T03:26:39.479 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:39 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-10T03:26:39.479 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:39 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:39.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:39 smithi164 ceph-mon[25711]: pgmap v234: 97 pgs: 14 unknown, 83 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-10T03:26:39.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:39 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-10T03:26:39.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:39 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-10T03:26:39.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:39 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:39.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:39 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.wfpoty", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-10T03:26:39.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:39 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.wfpoty", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-10T03:26:39.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:39 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-10T03:26:39.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:39 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-10T03:26:39.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:39 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:40.494 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:40 smithi130 ceph-mon[23614]: from='client.14536 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:40.495 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:40 smithi130 ceph-mon[23614]: Creating key for client.nfs.foo.1.0.smithi164.wfpoty 2024-09-10T03:26:40.495 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:40 smithi130 ceph-mon[23614]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-10T03:26:40.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:40 smithi164 ceph-mon[25711]: from='client.14536 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:40.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:40 smithi164 ceph-mon[25711]: Creating key for client.nfs.foo.1.0.smithi164.wfpoty 2024-09-10T03:26:40.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:40 smithi164 ceph-mon[25711]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-10T03:26:41.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:41 smithi130 ceph-mon[23614]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 159 B/s rd, 318 B/s wr, 0 op/s 2024-09-10T03:26:41.514 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:41 smithi164 ceph-mon[25711]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 159 B/s rd, 318 B/s wr, 0 op/s 2024-09-10T03:26:42.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:42 smithi164 ceph-mon[25711]: from='client.14552 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:42.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-10T03:26:42.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-10T03:26:42.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.wfpoty-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-10T03:26:42.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.wfpoty-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-10T03:26:42.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:42.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-09-10T03:26:42.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2024-09-10T03:26:42.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-09-10T03:26:42.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-10T03:26:42.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:26:42.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:26:42.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:42 smithi130 ceph-mon[23614]: from='client.14552 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:42.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-10T03:26:42.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-10T03:26:42.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.wfpoty-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-10T03:26:42.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.wfpoty-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-10T03:26:42.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:42.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-09-10T03:26:42.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2024-09-10T03:26:42.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-09-10T03:26:42.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-10T03:26:42.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:26:42.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:26:43.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:43 smithi130 ceph-mon[23614]: Rados config object exists: conf-nfs.foo 2024-09-10T03:26:43.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:43 smithi130 ceph-mon[23614]: Creating key for client.nfs.foo.1.0.smithi164.wfpoty-rgw 2024-09-10T03:26:43.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:43 smithi130 ceph-mon[23614]: Deploying daemon nfs.foo.1.0.smithi164.wfpoty on smithi164 2024-09-10T03:26:43.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:43 smithi130 ceph-mon[23614]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-09-10T03:26:43.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:43 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-09-10T03:26:43.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:43 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]': finished 2024-09-10T03:26:43.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:43 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-09-10T03:26:43.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:43 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-09-10T03:26:43.514 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:43 smithi130 ceph-mon[23614]: osdmap e69: 8 total, 8 up, 8 in 2024-09-10T03:26:43.565 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:43 smithi164 ceph-mon[25711]: Rados config object exists: conf-nfs.foo 2024-09-10T03:26:43.565 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:43 smithi164 ceph-mon[25711]: Creating key for client.nfs.foo.1.0.smithi164.wfpoty-rgw 2024-09-10T03:26:43.565 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:43 smithi164 ceph-mon[25711]: Deploying daemon nfs.foo.1.0.smithi164.wfpoty on smithi164 2024-09-10T03:26:43.565 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:43 smithi164 ceph-mon[25711]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-09-10T03:26:43.565 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:43 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-09-10T03:26:43.565 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:43 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]': finished 2024-09-10T03:26:43.565 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:43 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-09-10T03:26:43.565 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:43 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-09-10T03:26:43.565 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:43 smithi164 ceph-mon[25711]: osdmap e69: 8 total, 8 up, 8 in 2024-09-10T03:26:44.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:44 smithi130 ceph-mon[23614]: from='client.14566 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:44.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:44 smithi130 ceph-mon[23614]: osdmap e70: 8 total, 8 up, 8 in 2024-09-10T03:26:44.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:44 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:44.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:44 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:26:44.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:44 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:44.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:44 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:26:44.591 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:44 smithi164 ceph-mon[25711]: from='client.14566 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:44.591 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:44 smithi164 ceph-mon[25711]: osdmap e70: 8 total, 8 up, 8 in 2024-09-10T03:26:44.591 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:44 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:44.591 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:44 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:26:44.591 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:44 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:44.591 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:44 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:26:45.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:45 smithi130 ceph-mon[23614]: pgmap v239: 97 pgs: 2 peering, 95 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-10T03:26:45.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:45 smithi130 ceph-mon[23614]: from='client.14570 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:45.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:45 smithi164 ceph-mon[25711]: pgmap v239: 97 pgs: 2 peering, 95 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-10T03:26:45.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:45 smithi164 ceph-mon[25711]: from='client.14570 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:46.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:46 smithi164 ceph-mon[25711]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:46.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:46 smithi130 ceph-mon[23614]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:47.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:47 smithi130 ceph-mon[23614]: pgmap v240: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 69 B/s, 0 objects/s recovering 2024-09-10T03:26:47.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:47 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:47.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:47 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:47.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:47 smithi164 ceph-mon[25711]: pgmap v240: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 69 B/s, 0 objects/s recovering 2024-09-10T03:26:47.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:47 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:47.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:47 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:48.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:48 smithi130 ceph-mon[23614]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:48.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:48 smithi164 ceph-mon[25711]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:49.152 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo ?:2049 2/2 2s ago 14s count:2 2024-09-10T03:26:49.469 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:49 smithi130 ceph-mon[23614]: pgmap v241: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 69 B/s, 0 objects/s recovering 2024-09-10T03:26:49.469 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:49 smithi130 ceph-mon[23614]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:49.469 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:49 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:49.469 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:49 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:49.469 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:49 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:49.469 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:49 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:26:49.469 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:49 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:49.469 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:49 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:26:49.470 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:49 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:49.470 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:49 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:49.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:49 smithi164 ceph-mon[25711]: pgmap v241: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 69 B/s, 0 objects/s recovering 2024-09-10T03:26:49.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:49 smithi164 ceph-mon[25711]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:49.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:49 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:49.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:49 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:49.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:49 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:49.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:49 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:26:49.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:49 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:26:49.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:49 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:26:49.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:49 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:49.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:49 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:26:49.884 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-10T03:26:49.894 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi130.front.sepia.ceph.com 2024-09-10T03:26:49.894 DEBUG:teuthology.orchestra.run.smithi130:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-10T03:26:49.920 INFO:teuthology.orchestra.run.smithi130.stderr:+ mkdir /mnt/foo 2024-09-10T03:26:49.929 DEBUG:teuthology.orchestra.run.smithi130:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-10T03:26:49.993 INFO:teuthology.orchestra.run.smithi130.stderr:++ hostname 2024-09-10T03:26:49.994 INFO:teuthology.orchestra.run.smithi130.stderr:+ mount -t nfs smithi130:/fake /mnt/foo -o sync 2024-09-10T03:26:50.433 DEBUG:teuthology.orchestra.run.smithi130:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-10T03:26:50.495 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:50 smithi130 ceph-mon[23614]: from='client.14598 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:50.498 INFO:teuthology.orchestra.run.smithi130.stderr:+ echo test 2024-09-10T03:26:50.509 DEBUG:teuthology.orchestra.run.smithi130:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-10T03:26:50.579 INFO:teuthology.orchestra.run.smithi130.stderr:+ sync 2024-09-10T03:26:50.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:50 smithi164 ceph-mon[25711]: from='client.14598 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:26:50.775 INFO:teuthology.run_tasks:Running task parallel... 2024-09-10T03:26:50.785 INFO:teuthology.task.parallel:starting parallel... 2024-09-10T03:26:50.786 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-10T03:26:50.786 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-10T03:26:50.787 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi130.front.sepia.ceph.com 2024-09-10T03:26:50.787 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -e sha1=3a5e3f16720f432aea965c40a8f6eb480095ec6c -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-10T03:26:50.787 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-10T03:26:50.787 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-10T03:26:50.788 INFO:teuthology.task.exec:Executing custom commands... 2024-09-10T03:26:50.788 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi130.front.sepia.ceph.com 2024-09-10T03:26:50.789 DEBUG:teuthology.orchestra.run.smithi130:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-10T03:26:50.901 INFO:teuthology.orchestra.run.smithi130.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-10T03:26:50.901 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:26:50.901 INFO:teuthology.orchestra.run.smithi130.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-10T03:26:50.902 INFO:teuthology.orchestra.run.smithi130.stdout:4 of 5 processes prepared for launch 0 sec 2024-09-10T03:26:51.003 INFO:teuthology.orchestra.run.smithi130.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-10T03:26:51.003 INFO:teuthology.orchestra.run.smithi130.stdout:releasing clients 2024-09-10T03:26:51.312 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:51 smithi130 ceph-mon[23614]: pgmap v242: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 KiB/s wr, 6 op/s; 69 B/s, 0 objects/s recovering 2024-09-10T03:26:51.589 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:51 smithi164 ceph-mon[25711]: pgmap v242: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 KiB/s wr, 6 op/s; 69 B/s, 0 objects/s recovering 2024-09-10T03:26:52.003 INFO:teuthology.orchestra.run.smithi130.stdout: 5 133 29.14 MB/sec warmup 1 sec latency 51.467 ms 2024-09-10T03:26:53.004 INFO:teuthology.orchestra.run.smithi130.stdout: 5 265 24.91 MB/sec warmup 2 sec latency 31.101 ms 2024-09-10T03:26:53.491 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:53 smithi130 ceph-mon[23614]: pgmap v243: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.8 KiB/s wr, 5 op/s; 60 B/s, 0 objects/s recovering 2024-09-10T03:26:53.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:53 smithi164 ceph-mon[25711]: pgmap v243: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.8 KiB/s wr, 5 op/s; 60 B/s, 0 objects/s recovering 2024-09-10T03:26:54.003 INFO:teuthology.orchestra.run.smithi130.stdout: 5 394 23.30 MB/sec warmup 3 sec latency 33.068 ms 2024-09-10T03:26:54.333 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -e sha1=3a5e3f16720f432aea965c40a8f6eb480095ec6c -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-10T03:26:55.003 INFO:teuthology.orchestra.run.smithi130.stdout: 5 536 23.24 MB/sec warmup 4 sec latency 24.279 ms 2024-09-10T03:26:55.393 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:55 smithi130 ceph-mon[23614]: pgmap v244: 97 pgs: 97 active+clean; 12 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.1 MiB/s wr, 25 op/s; 54 B/s, 0 objects/s recovering 2024-09-10T03:26:55.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:55 smithi164 ceph-mon[25711]: pgmap v244: 97 pgs: 97 active+clean; 12 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.1 MiB/s wr, 25 op/s; 54 B/s, 0 objects/s recovering 2024-09-10T03:26:56.003 INFO:teuthology.orchestra.run.smithi130.stdout: 5 794 21.44 MB/sec warmup 5 sec latency 30.817 ms 2024-09-10T03:26:57.003 INFO:teuthology.orchestra.run.smithi130.stdout: 5 1252 19.71 MB/sec warmup 6 sec latency 53.729 ms 2024-09-10T03:26:57.416 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:57 smithi130 ceph-mon[23614]: pgmap v245: 97 pgs: 97 active+clean; 71 MiB data, 111 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 6.0 MiB/s wr, 279 op/s; 46 B/s, 0 objects/s recovering 2024-09-10T03:26:57.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:57 smithi164 ceph-mon[25711]: pgmap v245: 97 pgs: 97 active+clean; 71 MiB data, 111 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 6.0 MiB/s wr, 279 op/s; 46 B/s, 0 objects/s recovering 2024-09-10T03:26:58.003 INFO:teuthology.orchestra.run.smithi130.stdout: 5 2262 19.41 MB/sec warmup 7 sec latency 24.445 ms 2024-09-10T03:26:58.797 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -e sha1=3a5e3f16720f432aea965c40a8f6eb480095ec6c -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-10T03:26:59.003 INFO:teuthology.orchestra.run.smithi130.stdout: 5 2956 20.68 MB/sec warmup 8 sec latency 32.345 ms 2024-09-10T03:26:59.440 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:26:59 smithi130 ceph-mon[23614]: pgmap v246: 97 pgs: 97 active+clean; 84 MiB data, 132 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 7.1 MiB/s wr, 297 op/s 2024-09-10T03:26:59.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:26:59 smithi164 ceph-mon[25711]: pgmap v246: 97 pgs: 97 active+clean; 84 MiB data, 132 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 7.1 MiB/s wr, 297 op/s 2024-09-10T03:27:00.003 INFO:teuthology.orchestra.run.smithi130.stdout: 5 3468 20.42 MB/sec warmup 9 sec latency 43.187 ms 2024-09-10T03:27:01.003 INFO:teuthology.orchestra.run.smithi130.stdout: 5 3835 19.91 MB/sec warmup 10 sec latency 41.727 ms 2024-09-10T03:27:01.473 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:01 smithi130 ceph-mon[23614]: pgmap v247: 97 pgs: 97 active+clean; 110 MiB data, 187 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 9.9 MiB/s wr, 465 op/s 2024-09-10T03:27:01.526 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:01 smithi164 ceph-mon[25711]: pgmap v247: 97 pgs: 97 active+clean; 110 MiB data, 187 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 9.9 MiB/s wr, 465 op/s 2024-09-10T03:27:02.003 INFO:teuthology.orchestra.run.smithi130.stdout: 5 4130 18.28 MB/sec warmup 11 sec latency 79.985 ms 2024-09-10T03:27:03.003 INFO:teuthology.orchestra.run.smithi130.stdout: 5 4759 17.65 MB/sec warmup 12 sec latency 60.193 ms 2024-09-10T03:27:03.328 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -e sha1=3a5e3f16720f432aea965c40a8f6eb480095ec6c -- bash -c 'ceph mgr module enable nfs --force' 2024-09-10T03:27:03.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:03 smithi130 ceph-mon[23614]: pgmap v248: 97 pgs: 97 active+clean; 126 MiB data, 215 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 12 MiB/s wr, 641 op/s 2024-09-10T03:27:03.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:03 smithi164 ceph-mon[25711]: pgmap v248: 97 pgs: 97 active+clean; 126 MiB data, 215 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 12 MiB/s wr, 641 op/s 2024-09-10T03:27:04.003 INFO:teuthology.orchestra.run.smithi130.stdout: 5 5428 17.58 MB/sec warmup 13 sec latency 32.523 ms 2024-09-10T03:27:05.003 INFO:teuthology.orchestra.run.smithi130.stdout: 5 5962 16.95 MB/sec warmup 14 sec latency 38.911 ms 2024-09-10T03:27:05.286 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:05 smithi130 ceph-mon[23614]: pgmap v249: 97 pgs: 97 active+clean; 129 MiB data, 231 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 13 MiB/s wr, 666 op/s 2024-09-10T03:27:05.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:05 smithi164 ceph-mon[25711]: pgmap v249: 97 pgs: 97 active+clean; 129 MiB data, 231 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 13 MiB/s wr, 666 op/s 2024-09-10T03:27:06.004 INFO:teuthology.orchestra.run.smithi130.stdout: 5 6329 16.80 MB/sec warmup 15 sec latency 53.760 ms 2024-09-10T03:27:06.298 INFO:teuthology.orchestra.run.smithi130.stderr:module 'nfs' is already enabled 2024-09-10T03:27:06.305 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:06 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3395168053' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-10T03:27:06.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:06 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/3395168053' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-10T03:27:07.004 INFO:teuthology.orchestra.run.smithi130.stdout: 5 6762 16.93 MB/sec warmup 16 sec latency 43.135 ms 2024-09-10T03:27:07.339 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:07 smithi130 ceph-mon[23614]: pgmap v250: 97 pgs: 97 active+clean; 131 MiB data, 286 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 14 MiB/s wr, 911 op/s 2024-09-10T03:27:07.340 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:07 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3395168053' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-10T03:27:07.340 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:07 smithi130 ceph-mon[23614]: mgrmap e21: smithi130.axcmel(active, since 6m), standbys: smithi164.bkfwap 2024-09-10T03:27:07.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:07 smithi164 ceph-mon[25711]: pgmap v250: 97 pgs: 97 active+clean; 131 MiB data, 286 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 14 MiB/s wr, 911 op/s 2024-09-10T03:27:07.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:07 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/3395168053' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-10T03:27:07.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:07 smithi164 ceph-mon[25711]: mgrmap e21: smithi130.axcmel(active, since 6m), standbys: smithi164.bkfwap 2024-09-10T03:27:07.767 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -e sha1=3a5e3f16720f432aea965c40a8f6eb480095ec6c -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-10T03:27:08.004 INFO:teuthology.orchestra.run.smithi130.stdout: 5 7071 16.60 MB/sec warmup 17 sec latency 62.474 ms 2024-09-10T03:27:09.004 INFO:teuthology.orchestra.run.smithi130.stdout: 5 7392 16.27 MB/sec warmup 18 sec latency 44.222 ms 2024-09-10T03:27:09.385 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:09 smithi130 ceph-mon[23614]: pgmap v251: 97 pgs: 97 active+clean; 132 MiB data, 282 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 8.9 MiB/s wr, 667 op/s 2024-09-10T03:27:09.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:09 smithi164 ceph-mon[25711]: pgmap v251: 97 pgs: 97 active+clean; 132 MiB data, 282 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 8.9 MiB/s wr, 667 op/s 2024-09-10T03:27:10.004 INFO:teuthology.orchestra.run.smithi130.stdout: 5 7600 15.58 MB/sec warmup 19 sec latency 94.608 ms 2024-09-10T03:27:10.518 INFO:teuthology.orchestra.run.smithi130.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3a5e3f16720f432aea965c40a8f6eb480095ec6c 2024-09-10T03:27:11.004 INFO:teuthology.orchestra.run.smithi130.stdout: 5 7945 14.99 MB/sec warmup 20 sec latency 60.120 ms 2024-09-10T03:27:11.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:11 smithi130 ceph-mon[23614]: pgmap v252: 97 pgs: 97 active+clean; 135 MiB data, 316 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 9.2 MiB/s wr, 770 op/s 2024-09-10T03:27:11.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:11 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:27:11.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:11 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:27:11.430 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:11 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:27:11.430 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:11 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:27:11.430 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:11 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:27:11.430 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:11 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:27:11.589 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:11 smithi164 ceph-mon[25711]: pgmap v252: 97 pgs: 97 active+clean; 135 MiB data, 316 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 9.2 MiB/s wr, 770 op/s 2024-09-10T03:27:11.589 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:11 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:27:11.589 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:11 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:27:11.589 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:11 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:27:11.589 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:11 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:27:11.589 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:11 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:27:11.590 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:11 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:27:11.799 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-10T03:27:11.799 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi130.front.sepia.ceph.com 2024-09-10T03:27:11.800 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -e sha1=3a5e3f16720f432aea965c40a8f6eb480095ec6c -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-10T03:27:12.004 INFO:teuthology.orchestra.run.smithi130.stdout: 5 8250 14.60 MB/sec warmup 21 sec latency 51.042 ms 2024-09-10T03:27:12.443 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:12 smithi130 ceph-mon[23614]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a5e3f16720f432aea965c40a8f6eb480095ec6c", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:27:12.443 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:12 smithi130 ceph-mon[23614]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a5e3f16720f432aea965c40a8f6eb480095ec6c 2024-09-10T03:27:12.444 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:12 smithi130 ceph-mon[23614]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a5e3f16720f432aea965c40a8f6eb480095ec6c 2024-09-10T03:27:12.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:12 smithi164 ceph-mon[25711]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a5e3f16720f432aea965c40a8f6eb480095ec6c", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:27:12.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:12 smithi164 ceph-mon[25711]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a5e3f16720f432aea965c40a8f6eb480095ec6c 2024-09-10T03:27:12.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:12 smithi164 ceph-mon[25711]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a5e3f16720f432aea965c40a8f6eb480095ec6c 2024-09-10T03:27:13.004 INFO:teuthology.orchestra.run.smithi130.stdout: 5 8668 14.38 MB/sec warmup 22 sec latency 33.429 ms 2024-09-10T03:27:13.448 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:13 smithi130 ceph-mon[23614]: pgmap v253: 97 pgs: 97 active+clean; 142 MiB data, 354 MiB used, 715 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.6 MiB/s wr, 668 op/s 2024-09-10T03:27:13.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:13 smithi164 ceph-mon[25711]: pgmap v253: 97 pgs: 97 active+clean; 142 MiB data, 354 MiB used, 715 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.6 MiB/s wr, 668 op/s 2024-09-10T03:27:14.004 INFO:teuthology.orchestra.run.smithi130.stdout: 5 9116 14.13 MB/sec warmup 23 sec latency 41.296 ms 2024-09-10T03:27:15.004 INFO:teuthology.orchestra.run.smithi130.stdout: 5 9471 13.86 MB/sec warmup 24 sec latency 53.555 ms 2024-09-10T03:27:15.491 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:27:15.561 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:15 smithi130 ceph-mon[23614]: pgmap v254: 97 pgs: 97 active+clean; 142 MiB data, 362 MiB used, 715 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.7 MiB/s wr, 507 op/s 2024-09-10T03:27:15.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:15 smithi164 ceph-mon[25711]: pgmap v254: 97 pgs: 97 active+clean; 142 MiB data, 362 MiB used, 715 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.7 MiB/s wr, 507 op/s 2024-09-10T03:27:16.004 INFO:teuthology.orchestra.run.smithi130.stdout: 5 9897 13.93 MB/sec warmup 25 sec latency 32.155 ms 2024-09-10T03:27:16.570 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:27:16.570 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (5m) 27s ago 7m 24.2M - ba2b418f427c a430c6fad8b8 2024-09-10T03:27:16.570 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (6m) 27s ago 6m 6400k - 17.2.0 e1d6a67b021e 138ed3222ba5 2024-09-10T03:27:16.570 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (6m) 29s ago 6m 6405k - 17.2.0 e1d6a67b021e 3623793da55d 2024-09-10T03:27:16.570 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (5m) 27s ago 6m 52.8M - 8.3.5 dad864ee21e9 1e4f86d07930 2024-09-10T03:27:16.571 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (59s) 27s ago 58s 15.0M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:27:16.571 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (56s) 29s ago 56s 13.5M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:27:16.571 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:9283 running (8m) 27s ago 8m 462M - 17.2.0 e1d6a67b021e 197f48917434 2024-09-10T03:27:16.571 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283 running (6m) 29s ago 6m 412M - 17.2.0 e1d6a67b021e 91b74e6868e2 2024-09-10T03:27:16.571 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (8m) 27s ago 8m 46.5M 2048M 17.2.0 e1d6a67b021e f2b698da8d6e 2024-09-10T03:27:16.571 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (6m) 29s ago 6m 36.9M 2048M 17.2.0 e1d6a67b021e 7a928b0b045c 2024-09-10T03:27:16.571 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (38s) 27s ago 38s 50.0M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:27:16.571 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (33s) 29s ago 32s 50.8M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:27:16.571 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (6m) 27s ago 6m 17.4M - 1dbe0e931976 fcb1f0cd878a 2024-09-10T03:27:16.571 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (6m) 29s ago 6m 17.0M - 1dbe0e931976 1b5c522b94af 2024-09-10T03:27:16.571 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (5m) 27s ago 5m 43.2M 3216M 17.2.0 e1d6a67b021e 1d66043abecd 2024-09-10T03:27:16.571 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (4m) 27s ago 4m 42.9M 3216M 17.2.0 e1d6a67b021e ca1ec04aecfa 2024-09-10T03:27:16.571 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (4m) 27s ago 4m 41.0M 3216M 17.2.0 e1d6a67b021e 5883f5d5e969 2024-09-10T03:27:16.572 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (4m) 27s ago 4m 42.3M 3216M 17.2.0 e1d6a67b021e b32084e1a404 2024-09-10T03:27:16.572 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (3m) 29s ago 3m 39.1M 3984M 17.2.0 e1d6a67b021e 3b5c9f00f64d 2024-09-10T03:27:16.572 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (3m) 29s ago 3m 40.5M 3984M 17.2.0 e1d6a67b021e f16357ac331e 2024-09-10T03:27:16.572 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (2m) 29s ago 2m 39.5M 3984M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:27:16.572 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (2m) 29s ago 2m 43.0M 3984M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:27:16.572 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (5m) 27s ago 5m 62.8M - 514e6a882f6e a405bca101aa 2024-09-10T03:27:17.004 INFO:teuthology.orchestra.run.smithi130.stdout: 5 10392 14.17 MB/sec warmup 26 sec latency 50.840 ms 2024-09-10T03:27:17.216 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:27:17.216 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:27:17.216 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:27:17.216 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:27:17.216 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:27:17.216 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:27:17.216 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:27:17.216 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:27:17.217 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-10T03:27:17.217 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:27:17.217 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:27:17.217 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:27:17.217 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:27:17.217 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:27:17.217 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-10T03:27:17.217 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:27:17.217 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:27:17.592 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:17 smithi130 ceph-mon[23614]: from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:27:17.592 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:17 smithi130 ceph-mon[23614]: pgmap v255: 97 pgs: 97 active+clean; 143 MiB data, 400 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 676 op/s 2024-09-10T03:27:17.592 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:17 smithi130 ceph-mon[23614]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:27:17.592 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:17 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/1817158127' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-10T03:27:17.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:17 smithi164 ceph-mon[25711]: from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:27:17.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:17 smithi164 ceph-mon[25711]: pgmap v255: 97 pgs: 97 active+clean; 143 MiB data, 400 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 676 op/s 2024-09-10T03:27:17.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:17 smithi164 ceph-mon[25711]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:27:17.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:17 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/1817158127' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-10T03:27:17.710 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:27:17.710 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3a5e3f16720f432aea965c40a8f6eb480095ec6c", 2024-09-10T03:27:17.710 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:27:17.711 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-09-10T03:27:17.711 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "", 2024-09-10T03:27:17.711 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:3a5e3f16720f432aea965c40a8f6eb480095ec6c image" 2024-09-10T03:27:17.711 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:27:18.004 INFO:teuthology.orchestra.run.smithi130.stdout: 5 10837 14.36 MB/sec warmup 27 sec latency 40.148 ms 2024-09-10T03:27:18.229 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-09-10T03:27:18.619 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:18 smithi130 ceph-mon[23614]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:27:18.619 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:18 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/2459532202' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-10T03:27:18.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:18 smithi164 ceph-mon[25711]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:27:18.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:18 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/2459532202' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-10T03:27:19.004 INFO:teuthology.orchestra.run.smithi130.stdout: 5 11103 13.98 MB/sec warmup 28 sec latency 46.534 ms 2024-09-10T03:27:19.625 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:19 smithi130 ceph-mon[23614]: pgmap v256: 97 pgs: 97 active+clean; 139 MiB data, 404 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.5 MiB/s wr, 425 op/s 2024-09-10T03:27:19.626 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:19 smithi130 ceph-mon[23614]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:27:19.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:19 smithi164 ceph-mon[25711]: pgmap v256: 97 pgs: 97 active+clean; 139 MiB data, 404 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.5 MiB/s wr, 425 op/s 2024-09-10T03:27:19.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:19 smithi164 ceph-mon[25711]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:27:20.004 INFO:teuthology.orchestra.run.smithi130.stdout: 5 11442 13.61 MB/sec warmup 29 sec latency 53.624 ms 2024-09-10T03:27:21.004 INFO:teuthology.orchestra.run.smithi130.stdout: 5 11782 13.41 MB/sec warmup 30 sec latency 64.181 ms 2024-09-10T03:27:21.438 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:21 smithi130 ceph-mon[23614]: pgmap v257: 97 pgs: 97 active+clean; 145 MiB data, 441 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 481 op/s 2024-09-10T03:27:21.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:21 smithi164 ceph-mon[25711]: pgmap v257: 97 pgs: 97 active+clean; 145 MiB data, 441 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 481 op/s 2024-09-10T03:27:22.004 INFO:teuthology.orchestra.run.smithi130.stdout: 5 12069 13.26 MB/sec warmup 31 sec latency 69.434 ms 2024-09-10T03:27:23.005 INFO:teuthology.orchestra.run.smithi130.stdout: 5 12495 13.11 MB/sec warmup 32 sec latency 40.750 ms 2024-09-10T03:27:23.500 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:23 smithi130 ceph-mon[23614]: pgmap v258: 97 pgs: 97 active+clean; 160 MiB data, 467 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.0 MiB/s wr, 555 op/s 2024-09-10T03:27:23.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:23 smithi164 ceph-mon[25711]: pgmap v258: 97 pgs: 97 active+clean; 160 MiB data, 467 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.0 MiB/s wr, 555 op/s 2024-09-10T03:27:24.004 INFO:teuthology.orchestra.run.smithi130.stdout: 5 12883 12.86 MB/sec warmup 33 sec latency 41.934 ms 2024-09-10T03:27:25.005 INFO:teuthology.orchestra.run.smithi130.stdout: 5 13249 12.77 MB/sec warmup 34 sec latency 43.081 ms 2024-09-10T03:27:25.564 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:25 smithi130 ceph-mon[23614]: pgmap v259: 97 pgs: 97 active+clean; 157 MiB data, 456 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.0 MiB/s wr, 501 op/s 2024-09-10T03:27:25.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:25 smithi164 ceph-mon[25711]: pgmap v259: 97 pgs: 97 active+clean; 157 MiB data, 456 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.0 MiB/s wr, 501 op/s 2024-09-10T03:27:26.005 INFO:teuthology.orchestra.run.smithi130.stdout: 5 13820 13.14 MB/sec warmup 35 sec latency 51.606 ms 2024-09-10T03:27:27.005 INFO:teuthology.orchestra.run.smithi130.stdout: 5 14220 13.19 MB/sec warmup 36 sec latency 61.332 ms 2024-09-10T03:27:27.628 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:27 smithi130 ceph-mon[23614]: pgmap v260: 97 pgs: 97 active+clean; 154 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 653 op/s 2024-09-10T03:27:27.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:27 smithi164 ceph-mon[25711]: pgmap v260: 97 pgs: 97 active+clean; 154 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 653 op/s 2024-09-10T03:27:28.005 INFO:teuthology.orchestra.run.smithi130.stdout: 5 14627 13.16 MB/sec warmup 37 sec latency 33.093 ms 2024-09-10T03:27:29.005 INFO:teuthology.orchestra.run.smithi130.stdout: 5 15134 12.94 MB/sec warmup 38 sec latency 43.473 ms 2024-09-10T03:27:29.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:29 smithi130 ceph-mon[23614]: pgmap v261: 97 pgs: 97 active+clean; 153 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-09-10T03:27:29.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:29 smithi164 ceph-mon[25711]: pgmap v261: 97 pgs: 97 active+clean; 153 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-09-10T03:27:30.005 INFO:teuthology.orchestra.run.smithi130.stdout: 5 15372 12.77 MB/sec warmup 39 sec latency 68.935 ms 2024-09-10T03:27:31.007 INFO:teuthology.orchestra.run.smithi130.stdout: 5 15704 12.68 MB/sec warmup 40 sec latency 61.589 ms 2024-09-10T03:27:31.470 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:31 smithi130 ceph-mon[23614]: pgmap v262: 97 pgs: 97 active+clean; 161 MiB data, 526 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.0 MiB/s wr, 517 op/s 2024-09-10T03:27:31.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:31 smithi164 ceph-mon[25711]: pgmap v262: 97 pgs: 97 active+clean; 161 MiB data, 526 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.0 MiB/s wr, 517 op/s 2024-09-10T03:27:32.007 INFO:teuthology.orchestra.run.smithi130.stdout: 5 16062 12.57 MB/sec warmup 41 sec latency 45.865 ms 2024-09-10T03:27:33.007 INFO:teuthology.orchestra.run.smithi130.stdout: 5 16478 12.39 MB/sec warmup 42 sec latency 39.994 ms 2024-09-10T03:27:33.524 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:33 smithi130 ceph-mon[23614]: pgmap v263: 97 pgs: 97 active+clean; 172 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.4 MiB/s wr, 635 op/s 2024-09-10T03:27:33.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:33 smithi164 ceph-mon[25711]: pgmap v263: 97 pgs: 97 active+clean; 172 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.4 MiB/s wr, 635 op/s 2024-09-10T03:27:34.007 INFO:teuthology.orchestra.run.smithi130.stdout: 5 16943 12.51 MB/sec warmup 43 sec latency 45.833 ms 2024-09-10T03:27:35.007 INFO:teuthology.orchestra.run.smithi130.stdout: 5 17456 12.70 MB/sec warmup 44 sec latency 42.389 ms 2024-09-10T03:27:35.559 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:35 smithi130 ceph-mon[23614]: pgmap v264: 97 pgs: 97 active+clean; 169 MiB data, 574 MiB used, 715 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.9 MiB/s wr, 464 op/s 2024-09-10T03:27:35.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:35 smithi164 ceph-mon[25711]: pgmap v264: 97 pgs: 97 active+clean; 169 MiB data, 574 MiB used, 715 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.9 MiB/s wr, 464 op/s 2024-09-10T03:27:36.007 INFO:teuthology.orchestra.run.smithi130.stdout: 5 17843 12.77 MB/sec warmup 45 sec latency 61.822 ms 2024-09-10T03:27:37.007 INFO:teuthology.orchestra.run.smithi130.stdout: 5 18098 12.65 MB/sec warmup 46 sec latency 72.782 ms 2024-09-10T03:27:37.604 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:37 smithi130 ceph-mon[23614]: pgmap v265: 97 pgs: 97 active+clean; 168 MiB data, 603 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 588 op/s 2024-09-10T03:27:37.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:37 smithi164 ceph-mon[25711]: pgmap v265: 97 pgs: 97 active+clean; 168 MiB data, 603 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 588 op/s 2024-09-10T03:27:38.007 INFO:teuthology.orchestra.run.smithi130.stdout: 5 18323 12.42 MB/sec warmup 47 sec latency 64.221 ms 2024-09-10T03:27:38.622 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:38 smithi130 ceph-mon[23614]: pgmap v266: 97 pgs: 97 active+clean; 169 MiB data, 608 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 441 op/s 2024-09-10T03:27:38.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:38 smithi164 ceph-mon[25711]: pgmap v266: 97 pgs: 97 active+clean; 169 MiB data, 608 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 441 op/s 2024-09-10T03:27:39.007 INFO:teuthology.orchestra.run.smithi130.stdout: 5 18777 12.32 MB/sec warmup 48 sec latency 64.932 ms 2024-09-10T03:27:40.007 INFO:teuthology.orchestra.run.smithi130.stdout: 5 19383 12.37 MB/sec warmup 49 sec latency 48.123 ms 2024-09-10T03:27:41.007 INFO:teuthology.orchestra.run.smithi130.stdout: 5 20053 12.31 MB/sec warmup 50 sec latency 52.712 ms 2024-09-10T03:27:41.153 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:41 smithi130 ceph-mon[23614]: pgmap v267: 97 pgs: 97 active+clean; 178 MiB data, 628 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.3 MiB/s wr, 548 op/s 2024-09-10T03:27:41.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:41 smithi164 ceph-mon[25711]: pgmap v267: 97 pgs: 97 active+clean; 178 MiB data, 628 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.3 MiB/s wr, 548 op/s 2024-09-10T03:27:42.007 INFO:teuthology.orchestra.run.smithi130.stdout: 5 20400 12.29 MB/sec warmup 51 sec latency 37.022 ms 2024-09-10T03:27:42.173 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:27:42.174 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:27:42.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:27:42.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:27:43.007 INFO:teuthology.orchestra.run.smithi130.stdout: 5 20951 12.53 MB/sec warmup 52 sec latency 75.215 ms 2024-09-10T03:27:43.377 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:43 smithi130 ceph-mon[23614]: pgmap v268: 97 pgs: 97 active+clean; 186 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.5 MiB/s wr, 693 op/s 2024-09-10T03:27:43.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:43 smithi164 ceph-mon[25711]: pgmap v268: 97 pgs: 97 active+clean; 186 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.5 MiB/s wr, 693 op/s 2024-09-10T03:27:44.007 INFO:teuthology.orchestra.run.smithi130.stdout: 5 21384 12.62 MB/sec warmup 53 sec latency 61.352 ms 2024-09-10T03:27:45.007 INFO:teuthology.orchestra.run.smithi130.stdout: 5 21700 12.54 MB/sec warmup 54 sec latency 91.939 ms 2024-09-10T03:27:45.150 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:45 smithi130 ceph-mon[23614]: pgmap v269: 97 pgs: 97 active+clean; 185 MiB data, 683 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.4 MiB/s wr, 531 op/s 2024-09-10T03:27:45.402 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:45 smithi164 ceph-mon[25711]: pgmap v269: 97 pgs: 97 active+clean; 185 MiB data, 683 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.4 MiB/s wr, 531 op/s 2024-09-10T03:27:46.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 21876 12.35 MB/sec warmup 55 sec latency 75.028 ms 2024-09-10T03:27:46.434 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:46 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:27:46.434 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:46 smithi130 ceph-mon[23614]: Upgrade: Target is version 19.3.0-4885-g3a5e3f16 (unknown) 2024-09-10T03:27:46.434 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:46 smithi130 ceph-mon[23614]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca'] 2024-09-10T03:27:46.435 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:46 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:27:46.435 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:46 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:27:46.435 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:46 smithi130 ceph-mon[23614]: Upgrade: Need to upgrade myself (mgr.smithi130.axcmel) 2024-09-10T03:27:46.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:46 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:27:46.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:46 smithi164 ceph-mon[25711]: Upgrade: Target is version 19.3.0-4885-g3a5e3f16 (unknown) 2024-09-10T03:27:46.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:46 smithi164 ceph-mon[25711]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca'] 2024-09-10T03:27:46.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:46 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:27:46.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:46 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:27:46.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:46 smithi164 ceph-mon[25711]: Upgrade: Need to upgrade myself (mgr.smithi130.axcmel) 2024-09-10T03:27:47.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 22124 12.18 MB/sec warmup 56 sec latency 72.995 ms 2024-09-10T03:27:47.458 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:47 smithi130 ceph-mon[23614]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca on smithi164 2024-09-10T03:27:47.458 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:47 smithi130 ceph-mon[23614]: pgmap v270: 97 pgs: 97 active+clean; 188 MiB data, 743 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 683 op/s 2024-09-10T03:27:47.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:47 smithi164 ceph-mon[25711]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca on smithi164 2024-09-10T03:27:47.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:47 smithi164 ceph-mon[25711]: pgmap v270: 97 pgs: 97 active+clean; 188 MiB data, 743 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 683 op/s 2024-09-10T03:27:48.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 22432 12.09 MB/sec warmup 57 sec latency 81.386 ms 2024-09-10T03:27:48.731 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:27:49.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 22749 12.04 MB/sec warmup 58 sec latency 46.462 ms 2024-09-10T03:27:49.496 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:49 smithi130 ceph-mon[23614]: pgmap v271: 97 pgs: 97 active+clean; 192 MiB data, 738 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.0 MiB/s wr, 560 op/s 2024-09-10T03:27:49.569 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:27:49.569 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (6m) 60s ago 7m 24.2M - ba2b418f427c a430c6fad8b8 2024-09-10T03:27:49.569 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (7m) 60s ago 7m 6400k - 17.2.0 e1d6a67b021e 138ed3222ba5 2024-09-10T03:27:49.569 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (6m) 62s ago 6m 6405k - 17.2.0 e1d6a67b021e 3623793da55d 2024-09-10T03:27:49.569 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (5m) 60s ago 7m 52.8M - 8.3.5 dad864ee21e9 1e4f86d07930 2024-09-10T03:27:49.570 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (92s) 60s ago 91s 15.0M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:27:49.570 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (89s) 62s ago 89s 13.5M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:27:49.570 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:9283 running (8m) 60s ago 8m 462M - 17.2.0 e1d6a67b021e 197f48917434 2024-09-10T03:27:49.570 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283 running (6m) 62s ago 6m 412M - 17.2.0 e1d6a67b021e 91b74e6868e2 2024-09-10T03:27:49.570 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (8m) 60s ago 8m 46.5M 2048M 17.2.0 e1d6a67b021e f2b698da8d6e 2024-09-10T03:27:49.570 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (6m) 62s ago 6m 36.9M 2048M 17.2.0 e1d6a67b021e 7a928b0b045c 2024-09-10T03:27:49.570 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (71s) 60s ago 71s 50.0M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:27:49.570 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (66s) 62s ago 65s 50.8M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:27:49.570 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (7m) 60s ago 7m 17.4M - 1dbe0e931976 fcb1f0cd878a 2024-09-10T03:27:49.570 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (6m) 62s ago 6m 17.0M - 1dbe0e931976 1b5c522b94af 2024-09-10T03:27:49.570 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (6m) 60s ago 6m 43.2M 3216M 17.2.0 e1d6a67b021e 1d66043abecd 2024-09-10T03:27:49.570 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (5m) 60s ago 5m 42.9M 3216M 17.2.0 e1d6a67b021e ca1ec04aecfa 2024-09-10T03:27:49.570 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (5m) 60s ago 5m 41.0M 3216M 17.2.0 e1d6a67b021e 5883f5d5e969 2024-09-10T03:27:49.570 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (4m) 60s ago 4m 42.3M 3216M 17.2.0 e1d6a67b021e b32084e1a404 2024-09-10T03:27:49.571 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (4m) 62s ago 4m 39.1M 3984M 17.2.0 e1d6a67b021e 3b5c9f00f64d 2024-09-10T03:27:49.571 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (3m) 62s ago 3m 40.5M 3984M 17.2.0 e1d6a67b021e f16357ac331e 2024-09-10T03:27:49.571 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (3m) 62s ago 3m 39.5M 3984M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:27:49.571 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (3m) 62s ago 3m 43.0M 3984M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:27:49.571 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (6m) 60s ago 6m 62.8M - 514e6a882f6e a405bca101aa 2024-09-10T03:27:49.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:49 smithi164 ceph-mon[25711]: pgmap v271: 97 pgs: 97 active+clean; 192 MiB data, 738 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.0 MiB/s wr, 560 op/s 2024-09-10T03:27:50.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 23143 11.93 MB/sec warmup 59 sec latency 49.240 ms 2024-09-10T03:27:50.029 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:27:50.030 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:27:50.030 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:27:50.030 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:27:50.030 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:27:50.030 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:27:50.030 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:27:50.030 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:27:50.030 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-10T03:27:50.030 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:27:50.030 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:27:50.030 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:27:50.030 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:27:50.030 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:27:50.030 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-10T03:27:50.031 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:27:50.031 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:27:50.521 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:50 smithi130 ceph-mon[23614]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:27:50.521 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:50 smithi130 ceph-mon[23614]: from='client.14650 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:27:50.522 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:50 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3233677939' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-10T03:27:50.538 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:27:50.538 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:27:50.539 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:27:50.539 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-09-10T03:27:50.539 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "0/23 daemons upgraded", 2024-09-10T03:27:50.539 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca image on host smithi164" 2024-09-10T03:27:50.539 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:27:50.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:50 smithi164 ceph-mon[25711]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:27:50.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:50 smithi164 ceph-mon[25711]: from='client.14650 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:27:50.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:50 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/3233677939' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-10T03:27:50.990 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-09-10T03:27:51.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 23611 11.86 MB/sec warmup 60 sec latency 95.363 ms 2024-09-10T03:27:51.538 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:51 smithi130 ceph-mon[23614]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:27:51.539 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:51 smithi130 ceph-mon[23614]: pgmap v272: 97 pgs: 97 active+clean; 198 MiB data, 766 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.0 MiB/s wr, 692 op/s 2024-09-10T03:27:51.539 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:51 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/166909871' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-10T03:27:51.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:51 smithi164 ceph-mon[25711]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:27:51.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:51 smithi164 ceph-mon[25711]: pgmap v272: 97 pgs: 97 active+clean; 198 MiB data, 766 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.0 MiB/s wr, 692 op/s 2024-09-10T03:27:51.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:51 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/166909871' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-10T03:27:52.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 23999 11.89 MB/sec warmup 61 sec latency 41.072 ms 2024-09-10T03:27:52.575 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:52 smithi130 ceph-mon[23614]: from='client.24405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:27:52.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:52 smithi164 ceph-mon[25711]: from='client.24405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:27:53.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 24362 11.97 MB/sec warmup 62 sec latency 69.639 ms 2024-09-10T03:27:53.590 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:53 smithi130 ceph-mon[23614]: pgmap v273: 97 pgs: 97 active+clean; 190 MiB data, 794 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 612 op/s 2024-09-10T03:27:53.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:53 smithi164 ceph-mon[25711]: pgmap v273: 97 pgs: 97 active+clean; 190 MiB data, 794 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 612 op/s 2024-09-10T03:27:54.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 24863 12.08 MB/sec warmup 63 sec latency 65.047 ms 2024-09-10T03:27:55.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 25211 12.06 MB/sec warmup 64 sec latency 69.817 ms 2024-09-10T03:27:55.612 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:55 smithi130 ceph-mon[23614]: pgmap v274: 97 pgs: 97 active+clean; 195 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 539 op/s 2024-09-10T03:27:55.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:55 smithi164 ceph-mon[25711]: pgmap v274: 97 pgs: 97 active+clean; 195 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 539 op/s 2024-09-10T03:27:56.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 25542 11.94 MB/sec warmup 65 sec latency 43.780 ms 2024-09-10T03:27:57.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 26019 11.89 MB/sec warmup 66 sec latency 64.046 ms 2024-09-10T03:27:57.392 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:57 smithi130 ceph-mon[23614]: pgmap v275: 97 pgs: 97 active+clean; 201 MiB data, 820 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.3 MiB/s wr, 575 op/s 2024-09-10T03:27:57.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:57 smithi164 ceph-mon[25711]: pgmap v275: 97 pgs: 97 active+clean; 201 MiB data, 820 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.3 MiB/s wr, 575 op/s 2024-09-10T03:27:58.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 26680 11.97 MB/sec warmup 67 sec latency 32.385 ms 2024-09-10T03:27:59.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 27326 11.95 MB/sec warmup 68 sec latency 32.476 ms 2024-09-10T03:27:59.413 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:27:59 smithi130 ceph-mon[23614]: pgmap v276: 97 pgs: 97 active+clean; 202 MiB data, 820 MiB used, 714 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.6 MiB/s wr, 404 op/s 2024-09-10T03:27:59.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:27:59 smithi164 ceph-mon[25711]: pgmap v276: 97 pgs: 97 active+clean; 202 MiB data, 820 MiB used, 714 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.6 MiB/s wr, 404 op/s 2024-09-10T03:28:00.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 27863 12.09 MB/sec warmup 69 sec latency 118.857 ms 2024-09-10T03:28:01.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 28321 12.17 MB/sec warmup 70 sec latency 48.612 ms 2024-09-10T03:28:01.501 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:01 smithi130 ceph-mon[23614]: pgmap v277: 97 pgs: 97 active+clean; 208 MiB data, 858 MiB used, 714 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.7 MiB/s wr, 575 op/s 2024-09-10T03:28:01.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:01 smithi164 ceph-mon[25711]: pgmap v277: 97 pgs: 97 active+clean; 208 MiB data, 858 MiB used, 714 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.7 MiB/s wr, 575 op/s 2024-09-10T03:28:02.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 28617 12.20 MB/sec warmup 71 sec latency 52.259 ms 2024-09-10T03:28:03.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 28826 12.07 MB/sec warmup 72 sec latency 51.143 ms 2024-09-10T03:28:03.579 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:03 smithi130 ceph-mon[23614]: pgmap v278: 97 pgs: 97 active+clean; 211 MiB data, 892 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.0 MiB/s wr, 588 op/s 2024-09-10T03:28:03.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:03 smithi164 ceph-mon[25711]: pgmap v278: 97 pgs: 97 active+clean; 211 MiB data, 892 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.0 MiB/s wr, 588 op/s 2024-09-10T03:28:04.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 29202 11.95 MB/sec warmup 73 sec latency 37.472 ms 2024-09-10T03:28:04.593 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:04 smithi130 ceph-mon[23614]: pgmap v279: 97 pgs: 97 active+clean; 211 MiB data, 897 MiB used, 714 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.2 MiB/s wr, 571 op/s 2024-09-10T03:28:04.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:04 smithi164 ceph-mon[25711]: pgmap v279: 97 pgs: 97 active+clean; 211 MiB data, 897 MiB used, 714 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.2 MiB/s wr, 571 op/s 2024-09-10T03:28:05.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 29573 11.91 MB/sec warmup 74 sec latency 74.756 ms 2024-09-10T03:28:06.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 29948 11.90 MB/sec warmup 75 sec latency 57.986 ms 2024-09-10T03:28:07.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 30414 11.82 MB/sec warmup 76 sec latency 60.569 ms 2024-09-10T03:28:07.162 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:07 smithi130 ceph-mon[23614]: pgmap v280: 97 pgs: 97 active+clean; 225 MiB data, 949 MiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.3 MiB/s wr, 665 op/s 2024-09-10T03:28:07.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:07 smithi164 ceph-mon[25711]: pgmap v280: 97 pgs: 97 active+clean; 225 MiB data, 949 MiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.3 MiB/s wr, 665 op/s 2024-09-10T03:28:08.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 30762 11.74 MB/sec warmup 77 sec latency 56.759 ms 2024-09-10T03:28:09.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 31054 11.75 MB/sec warmup 78 sec latency 55.440 ms 2024-09-10T03:28:09.218 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:09 smithi130 ceph-mon[23614]: pgmap v281: 97 pgs: 97 active+clean; 225 MiB data, 961 MiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.0 MiB/s wr, 632 op/s 2024-09-10T03:28:09.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:09 smithi164 ceph-mon[25711]: pgmap v281: 97 pgs: 97 active+clean; 225 MiB data, 961 MiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.0 MiB/s wr, 632 op/s 2024-09-10T03:28:10.013 INFO:teuthology.orchestra.run.smithi130.stdout: 5 31512 11.84 MB/sec warmup 79 sec latency 86.656 ms 2024-09-10T03:28:11.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 31901 11.86 MB/sec warmup 80 sec latency 43.573 ms 2024-09-10T03:28:11.265 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:11 smithi130 ceph-mon[23614]: pgmap v282: 97 pgs: 97 active+clean; 220 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.4 MiB/s wr, 666 op/s 2024-09-10T03:28:11.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:11 smithi164 ceph-mon[25711]: pgmap v282: 97 pgs: 97 active+clean; 220 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.4 MiB/s wr, 666 op/s 2024-09-10T03:28:12.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 32232 11.89 MB/sec warmup 81 sec latency 56.872 ms 2024-09-10T03:28:13.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 32651 11.79 MB/sec warmup 82 sec latency 64.932 ms 2024-09-10T03:28:13.328 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:13 smithi130 ceph-mon[23614]: pgmap v283: 97 pgs: 97 active+clean; 221 MiB data, 1003 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.0 MiB/s wr, 607 op/s 2024-09-10T03:28:13.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:13 smithi164 ceph-mon[25711]: pgmap v283: 97 pgs: 97 active+clean; 221 MiB data, 1003 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.0 MiB/s wr, 607 op/s 2024-09-10T03:28:14.008 INFO:teuthology.orchestra.run.smithi130.stdout: 5 33005 11.71 MB/sec warmup 83 sec latency 86.976 ms 2024-09-10T03:28:15.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 33313 11.69 MB/sec warmup 84 sec latency 71.271 ms 2024-09-10T03:28:15.382 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:15 smithi130 ceph-mon[23614]: pgmap v284: 97 pgs: 97 active+clean; 225 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 463 op/s 2024-09-10T03:28:15.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:15 smithi164 ceph-mon[25711]: pgmap v284: 97 pgs: 97 active+clean; 225 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 463 op/s 2024-09-10T03:28:16.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 33755 11.67 MB/sec warmup 85 sec latency 55.438 ms 2024-09-10T03:28:17.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 34199 11.59 MB/sec warmup 86 sec latency 50.174 ms 2024-09-10T03:28:17.185 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:17 smithi130 ceph-mon[23614]: pgmap v285: 97 pgs: 97 active+clean; 237 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 665 op/s 2024-09-10T03:28:17.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:17 smithi164 ceph-mon[25711]: pgmap v285: 97 pgs: 97 active+clean; 237 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 665 op/s 2024-09-10T03:28:18.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 34623 11.63 MB/sec warmup 87 sec latency 80.884 ms 2024-09-10T03:28:19.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 35397 11.88 MB/sec warmup 88 sec latency 159.089 ms 2024-09-10T03:28:19.238 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:19 smithi130 ceph-mon[23614]: pgmap v286: 97 pgs: 97 active+clean; 235 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.6 MiB/s wr, 455 op/s 2024-09-10T03:28:19.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:19 smithi164 ceph-mon[25711]: pgmap v286: 97 pgs: 97 active+clean; 235 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.6 MiB/s wr, 455 op/s 2024-09-10T03:28:20.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 35788 11.89 MB/sec warmup 89 sec latency 97.607 ms 2024-09-10T03:28:21.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 36087 11.79 MB/sec warmup 90 sec latency 40.717 ms 2024-09-10T03:28:21.294 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:21 smithi130 ceph-mon[23614]: pgmap v287: 97 pgs: 97 active+clean; 232 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.4 MiB/s wr, 469 op/s 2024-09-10T03:28:21.442 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:28:21.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:21 smithi164 ceph-mon[25711]: pgmap v287: 97 pgs: 97 active+clean; 232 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.4 MiB/s wr, 469 op/s 2024-09-10T03:28:22.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 36492 11.73 MB/sec warmup 91 sec latency 33.768 ms 2024-09-10T03:28:22.304 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:22 smithi130 ceph-mon[23614]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:28:22.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:22 smithi164 ceph-mon[25711]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:28:22.483 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:28:22.483 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (6m) 93s ago 8m 24.2M - ba2b418f427c a430c6fad8b8 2024-09-10T03:28:22.483 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (8m) 93s ago 8m 6400k - 17.2.0 e1d6a67b021e 138ed3222ba5 2024-09-10T03:28:22.483 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (7m) 95s ago 7m 6405k - 17.2.0 e1d6a67b021e 3623793da55d 2024-09-10T03:28:22.483 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (6m) 93s ago 7m 52.8M - 8.3.5 dad864ee21e9 1e4f86d07930 2024-09-10T03:28:22.483 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (2m) 93s ago 2m 15.0M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:28:22.483 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (2m) 95s ago 2m 13.5M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:28:22.483 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:9283 running (9m) 93s ago 9m 462M - 17.2.0 e1d6a67b021e 197f48917434 2024-09-10T03:28:22.483 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283 running (7m) 95s ago 7m 412M - 17.2.0 e1d6a67b021e 91b74e6868e2 2024-09-10T03:28:22.483 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (9m) 93s ago 9m 46.5M 2048M 17.2.0 e1d6a67b021e f2b698da8d6e 2024-09-10T03:28:22.483 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (7m) 95s ago 7m 36.9M 2048M 17.2.0 e1d6a67b021e 7a928b0b045c 2024-09-10T03:28:22.483 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (104s) 93s ago 103s 50.0M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:28:22.484 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (99s) 95s ago 98s 50.8M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:28:22.484 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (7m) 93s ago 7m 17.4M - 1dbe0e931976 fcb1f0cd878a 2024-09-10T03:28:22.484 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (7m) 95s ago 7m 17.0M - 1dbe0e931976 1b5c522b94af 2024-09-10T03:28:22.484 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (6m) 93s ago 6m 43.2M 3216M 17.2.0 e1d6a67b021e 1d66043abecd 2024-09-10T03:28:22.484 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (6m) 93s ago 6m 42.9M 3216M 17.2.0 e1d6a67b021e ca1ec04aecfa 2024-09-10T03:28:22.484 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (5m) 93s ago 5m 41.0M 3216M 17.2.0 e1d6a67b021e 5883f5d5e969 2024-09-10T03:28:22.484 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (5m) 93s ago 5m 42.3M 3216M 17.2.0 e1d6a67b021e b32084e1a404 2024-09-10T03:28:22.484 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (4m) 95s ago 4m 39.1M 3984M 17.2.0 e1d6a67b021e 3b5c9f00f64d 2024-09-10T03:28:22.484 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (4m) 95s ago 4m 40.5M 3984M 17.2.0 e1d6a67b021e f16357ac331e 2024-09-10T03:28:22.484 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (3m) 95s ago 3m 39.5M 3984M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:28:22.484 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (3m) 95s ago 3m 43.0M 3984M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:28:22.484 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (7m) 93s ago 7m 62.8M - 514e6a882f6e a405bca101aa 2024-09-10T03:28:23.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 37070 11.75 MB/sec warmup 92 sec latency 38.719 ms 2024-09-10T03:28:23.105 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:28:23.105 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:28:23.105 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:28:23.105 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:28:23.105 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:28:23.105 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:28:23.105 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:28:23.105 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:28:23.106 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-10T03:28:23.106 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:28:23.106 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:28:23.106 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:28:23.106 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:28:23.106 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:28:23.106 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-10T03:28:23.106 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:28:23.106 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:28:23.320 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:23 smithi130 ceph-mon[23614]: pgmap v288: 97 pgs: 97 active+clean; 241 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 644 op/s 2024-09-10T03:28:23.320 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:23 smithi130 ceph-mon[23614]: from='client.24413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:28:23.320 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:23 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:28:23.320 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:23 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.bkfwap", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-10T03:28:23.320 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:23 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-10T03:28:23.320 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:23 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:28:23.320 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:23 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/1732926484' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-10T03:28:23.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:23 smithi164 ceph-mon[25711]: pgmap v288: 97 pgs: 97 active+clean; 241 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 644 op/s 2024-09-10T03:28:23.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:23 smithi164 ceph-mon[25711]: from='client.24413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:28:23.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:23 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:28:23.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:23 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.bkfwap", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-10T03:28:23.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:23 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-10T03:28:23.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:23 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:28:23.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:23 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/1732926484' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-10T03:28:23.639 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:28:23.639 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:28:23.639 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:28:23.639 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-09-10T03:28:23.639 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "0/23 daemons upgraded", 2024-09-10T03:28:23.639 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading mgr daemons" 2024-09-10T03:28:23.639 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:28:24.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 37690 11.76 MB/sec warmup 93 sec latency 36.850 ms 2024-09-10T03:28:24.239 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-09-10T03:28:24.333 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:24 smithi130 ceph-mon[23614]: from='client.14678 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:28:24.333 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:24 smithi130 ceph-mon[23614]: Upgrade: Updating mgr.smithi164.bkfwap 2024-09-10T03:28:24.333 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:24 smithi130 ceph-mon[23614]: Deploying daemon mgr.smithi164.bkfwap on smithi164 2024-09-10T03:28:24.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:24 smithi164 ceph-mon[25711]: from='client.14678 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:28:24.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:24 smithi164 ceph-mon[25711]: Upgrade: Updating mgr.smithi164.bkfwap 2024-09-10T03:28:24.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:24 smithi164 ceph-mon[25711]: Deploying daemon mgr.smithi164.bkfwap on smithi164 2024-09-10T03:28:25.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 38197 11.81 MB/sec warmup 94 sec latency 146.539 ms 2024-09-10T03:28:25.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:25 smithi130 ceph-mon[23614]: from='client.24421 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:28:25.364 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:25 smithi130 ceph-mon[23614]: pgmap v289: 97 pgs: 97 active+clean; 247 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 6.4 MiB/s wr, 602 op/s 2024-09-10T03:28:25.364 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:25 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3017966292' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-10T03:28:25.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:25 smithi164 ceph-mon[25711]: from='client.24421 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:28:25.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:25 smithi164 ceph-mon[25711]: pgmap v289: 97 pgs: 97 active+clean; 247 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 6.4 MiB/s wr, 602 op/s 2024-09-10T03:28:25.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:25 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/3017966292' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-10T03:28:26.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 38627 11.85 MB/sec warmup 95 sec latency 66.693 ms 2024-09-10T03:28:27.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 38984 11.90 MB/sec warmup 96 sec latency 91.242 ms 2024-09-10T03:28:27.396 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:27 smithi130 ceph-mon[23614]: pgmap v290: 97 pgs: 97 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 752 op/s 2024-09-10T03:28:27.396 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:27 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:28:27.396 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:27 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:28:27.396 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:27 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:28:27.396 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:27 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:28:27.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:27 smithi164 ceph-mon[25711]: pgmap v290: 97 pgs: 97 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 752 op/s 2024-09-10T03:28:27.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:27 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:28:27.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:27 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:28:27.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:27 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:28:27.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:27 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:28:28.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 39352 11.89 MB/sec warmup 97 sec latency 38.669 ms 2024-09-10T03:28:29.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 39662 11.81 MB/sec warmup 98 sec latency 124.776 ms 2024-09-10T03:28:29.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:29 smithi130 ceph-mon[23614]: pgmap v291: 97 pgs: 97 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.4 MiB/s wr, 540 op/s 2024-09-10T03:28:29.497 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:29 smithi164 ceph-mon[25711]: pgmap v291: 97 pgs: 97 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.4 MiB/s wr, 540 op/s 2024-09-10T03:28:30.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 39918 11.72 MB/sec warmup 99 sec latency 42.360 ms 2024-09-10T03:28:31.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 40139 11.66 MB/sec warmup 100 sec latency 83.126 ms 2024-09-10T03:28:31.256 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:31 smithi130 ceph-mon[23614]: pgmap v292: 97 pgs: 97 active+clean; 250 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.7 MiB/s wr, 660 op/s 2024-09-10T03:28:31.608 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:31 smithi164 ceph-mon[25711]: pgmap v292: 97 pgs: 97 active+clean; 250 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.7 MiB/s wr, 660 op/s 2024-09-10T03:28:32.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 40418 11.61 MB/sec warmup 101 sec latency 63.058 ms 2024-09-10T03:28:33.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 40706 11.55 MB/sec warmup 102 sec latency 101.327 ms 2024-09-10T03:28:33.324 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:33 smithi130 ceph-mon[23614]: pgmap v293: 97 pgs: 97 active+clean; 261 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 725 op/s 2024-09-10T03:28:33.325 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:33 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/crt"}]: dispatch 2024-09-10T03:28:33.325 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:33 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-10T03:28:33.325 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:33 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/key"}]: dispatch 2024-09-10T03:28:33.325 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:33 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-10T03:28:33.325 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:33 smithi130 ceph-mon[23614]: Standby manager daemon smithi164.bkfwap restarted 2024-09-10T03:28:33.325 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:33 smithi130 ceph-mon[23614]: Standby manager daemon smithi164.bkfwap started 2024-09-10T03:28:33.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:33 smithi164 ceph-mon[25711]: pgmap v293: 97 pgs: 97 active+clean; 261 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 725 op/s 2024-09-10T03:28:33.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:33 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/crt"}]: dispatch 2024-09-10T03:28:33.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:33 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-10T03:28:33.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:33 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/key"}]: dispatch 2024-09-10T03:28:33.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:33 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-10T03:28:33.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:33 smithi164 ceph-mon[25711]: Standby manager daemon smithi164.bkfwap restarted 2024-09-10T03:28:33.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:33 smithi164 ceph-mon[25711]: Standby manager daemon smithi164.bkfwap started 2024-09-10T03:28:34.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 41135 11.53 MB/sec warmup 103 sec latency 62.600 ms 2024-09-10T03:28:34.369 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:34 smithi130 ceph-mon[23614]: mgrmap e22: smithi130.axcmel(active, since 7m), standbys: smithi164.bkfwap 2024-09-10T03:28:34.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:34 smithi164 ceph-mon[25711]: mgrmap e22: smithi130.axcmel(active, since 7m), standbys: smithi164.bkfwap 2024-09-10T03:28:35.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 41686 11.56 MB/sec warmup 104 sec latency 44.767 ms 2024-09-10T03:28:35.414 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:35 smithi130 ceph-mon[23614]: pgmap v294: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.6 MiB/s wr, 521 op/s 2024-09-10T03:28:35.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:35 smithi164 ceph-mon[25711]: pgmap v294: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.6 MiB/s wr, 521 op/s 2024-09-10T03:28:36.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 42063 11.59 MB/sec warmup 105 sec latency 62.580 ms 2024-09-10T03:28:37.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 42520 11.68 MB/sec warmup 106 sec latency 61.837 ms 2024-09-10T03:28:37.451 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:37 smithi130 ceph-mon[23614]: pgmap v295: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.1 MiB/s wr, 613 op/s 2024-09-10T03:28:37.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:37 smithi164 ceph-mon[25711]: pgmap v295: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.1 MiB/s wr, 613 op/s 2024-09-10T03:28:38.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 42753 11.64 MB/sec warmup 107 sec latency 96.567 ms 2024-09-10T03:28:39.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 43058 11.59 MB/sec warmup 108 sec latency 50.904 ms 2024-09-10T03:28:39.490 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:39 smithi130 ceph-mon[23614]: pgmap v296: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 456 op/s 2024-09-10T03:28:39.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:39 smithi164 ceph-mon[25711]: pgmap v296: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 456 op/s 2024-09-10T03:28:40.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 43296 11.51 MB/sec warmup 109 sec latency 88.588 ms 2024-09-10T03:28:41.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 43642 11.47 MB/sec warmup 110 sec latency 83.343 ms 2024-09-10T03:28:41.377 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:41 smithi130 ceph-mon[23614]: pgmap v297: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.6 MiB/s wr, 481 op/s 2024-09-10T03:28:41.502 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:41 smithi164 ceph-mon[25711]: pgmap v297: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.6 MiB/s wr, 481 op/s 2024-09-10T03:28:42.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 44110 11.45 MB/sec warmup 111 sec latency 43.084 ms 2024-09-10T03:28:42.384 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:28:42.384 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:28:42.384 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:42 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:28:42.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:28:42.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:28:42.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:42 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:28:43.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 44517 11.42 MB/sec warmup 112 sec latency 74.925 ms 2024-09-10T03:28:43.439 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:43 smithi130 ceph-mon[23614]: pgmap v298: 97 pgs: 97 active+clean; 272 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.5 MiB/s wr, 513 op/s 2024-09-10T03:28:43.439 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:43 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:28:43.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:43 smithi164 ceph-mon[25711]: pgmap v298: 97 pgs: 97 active+clean; 272 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.5 MiB/s wr, 513 op/s 2024-09-10T03:28:43.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:43 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:28:44.009 INFO:teuthology.orchestra.run.smithi130.stdout: 5 44936 11.41 MB/sec warmup 113 sec latency 63.527 ms 2024-09-10T03:28:45.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 45291 11.40 MB/sec warmup 114 sec latency 86.706 ms 2024-09-10T03:28:45.439 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:45 smithi164 ceph-mon[25711]: pgmap v299: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.8 MiB/s wr, 508 op/s 2024-09-10T03:28:45.492 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:45 smithi130 ceph-mon[23614]: pgmap v299: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.8 MiB/s wr, 508 op/s 2024-09-10T03:28:46.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 45679 11.45 MB/sec warmup 115 sec latency 51.664 ms 2024-09-10T03:28:47.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 46269 11.54 MB/sec warmup 116 sec latency 98.110 ms 2024-09-10T03:28:47.016 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:46 smithi130 ceph-mon[23614]: pgmap v300: 97 pgs: 97 active+clean; 271 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 5.8 MiB/s wr, 599 op/s 2024-09-10T03:28:47.016 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:46 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:28:47.017 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:46 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:28:47.017 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:46 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:28:47.017 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:46 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:28:47.017 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:46 smithi130 ceph-mon[23614]: Upgrade: Need to upgrade myself (mgr.smithi130.axcmel) 2024-09-10T03:28:47.017 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:46 smithi130 ceph-mon[23614]: Failing over to other MGR 2024-09-10T03:28:47.017 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:46 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr fail", "who": "smithi130.axcmel"}]: dispatch 2024-09-10T03:28:47.017 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:46 smithi130 ceph-mon[23614]: osdmap e71: 8 total, 8 up, 8 in 2024-09-10T03:28:47.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:46 smithi164 ceph-mon[25711]: pgmap v300: 97 pgs: 97 active+clean; 271 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 5.8 MiB/s wr, 599 op/s 2024-09-10T03:28:47.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:46 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:28:47.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:46 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:28:47.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:46 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' 2024-09-10T03:28:47.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:46 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:28:47.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:46 smithi164 ceph-mon[25711]: Upgrade: Need to upgrade myself (mgr.smithi130.axcmel) 2024-09-10T03:28:47.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:46 smithi164 ceph-mon[25711]: Failing over to other MGR 2024-09-10T03:28:47.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:46 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr fail", "who": "smithi130.axcmel"}]: dispatch 2024-09-10T03:28:47.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:46 smithi164 ceph-mon[25711]: osdmap e71: 8 total, 8 up, 8 in 2024-09-10T03:28:48.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 46668 11.52 MB/sec warmup 117 sec latency 40.352 ms 2024-09-10T03:28:48.058 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:48 smithi130 ceph-mon[23614]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "mgr fail", "who": "smithi130.axcmel"}]': finished 2024-09-10T03:28:48.058 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:48 smithi130 ceph-mon[23614]: mgrmap e23: smithi164.bkfwap(active, starting, since 1.04528s) 2024-09-10T03:28:48.058 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:48 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mon metadata", "id": "smithi130"}]: dispatch 2024-09-10T03:28:48.058 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:48 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:28:48.058 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:48 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi130.ukcxnb"}]: dispatch 2024-09-10T03:28:48.058 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:48 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi164.awiezo"}]: dispatch 2024-09-10T03:28:48.058 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:48 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mgr metadata", "who": "smithi164.bkfwap", "id": "smithi164.bkfwap"}]: dispatch 2024-09-10T03:28:48.058 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:48 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-10T03:28:48.059 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:48 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-10T03:28:48.059 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:48 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-10T03:28:48.059 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:48 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-10T03:28:48.059 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:48 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-10T03:28:48.059 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:48 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-10T03:28:48.059 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:48 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-10T03:28:48.059 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:48 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-10T03:28:48.059 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:48 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-10T03:28:48.059 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:48 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-10T03:28:48.059 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:48 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-10T03:28:48.059 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:48 smithi130 ceph-mon[23614]: Manager daemon smithi164.bkfwap is now available 2024-09-10T03:28:48.267 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:48 smithi164 ceph-mon[25711]: from='mgr.14200 172.21.15.130:0/3200949248' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "mgr fail", "who": "smithi130.axcmel"}]': finished 2024-09-10T03:28:48.267 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:48 smithi164 ceph-mon[25711]: mgrmap e23: smithi164.bkfwap(active, starting, since 1.04528s) 2024-09-10T03:28:48.267 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:48 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mon metadata", "id": "smithi130"}]: dispatch 2024-09-10T03:28:48.267 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:48 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:28:48.267 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:48 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi130.ukcxnb"}]: dispatch 2024-09-10T03:28:48.268 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:48 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi164.awiezo"}]: dispatch 2024-09-10T03:28:48.268 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:48 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mgr metadata", "who": "smithi164.bkfwap", "id": "smithi164.bkfwap"}]: dispatch 2024-09-10T03:28:48.268 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:48 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-10T03:28:48.268 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:48 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-10T03:28:48.268 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:48 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-10T03:28:48.268 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:48 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-10T03:28:48.268 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:48 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-10T03:28:48.268 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:48 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-10T03:28:48.268 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:48 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-10T03:28:48.268 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:48 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-10T03:28:48.268 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:48 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-10T03:28:48.268 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:48 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-10T03:28:48.268 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:48 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-10T03:28:48.269 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:48 smithi164 ceph-mon[25711]: Manager daemon smithi164.bkfwap is now available 2024-09-10T03:28:49.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 46886 11.44 MB/sec warmup 118 sec latency 100.892 ms 2024-09-10T03:28:49.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:49 smithi130 ceph-mon[23614]: mgrmap e24: smithi164.bkfwap(active, since 2s) 2024-09-10T03:28:49.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:49 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:49.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:49 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:49.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:49 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:49.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:49 smithi130 ceph-mon[23614]: Checking for cert/key for grafana.smithi130 2024-09-10T03:28:49.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:49 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:49.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:49 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:28:49.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:49 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:28:49.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:49 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:28:49.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:49 smithi130 ceph-mon[23614]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:28:49.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:49 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi164.bkfwap/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:28:49.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:49 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi164.bkfwap/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:28:49.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:49 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi164.bkfwap/trash_purge_schedule"}]: dispatch 2024-09-10T03:28:49.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:49 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi164.bkfwap/trash_purge_schedule"}]: dispatch 2024-09-10T03:28:49.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:49 smithi130 ceph-mon[23614]: Deploying cephadm binary to smithi130 2024-09-10T03:28:49.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:49 smithi130 ceph-mon[23614]: Deploying cephadm binary to smithi164 2024-09-10T03:28:49.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:49 smithi164 ceph-mon[25711]: mgrmap e24: smithi164.bkfwap(active, since 2s) 2024-09-10T03:28:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:49 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:49 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:49 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:49 smithi164 ceph-mon[25711]: Checking for cert/key for grafana.smithi130 2024-09-10T03:28:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:49 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:49 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:28:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:49 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:28:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:49 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:28:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:49 smithi164 ceph-mon[25711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:28:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:49 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi164.bkfwap/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:28:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:49 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi164.bkfwap/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:28:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:49 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi164.bkfwap/trash_purge_schedule"}]: dispatch 2024-09-10T03:28:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:49 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi164.bkfwap/trash_purge_schedule"}]: dispatch 2024-09-10T03:28:49.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:49 smithi164 ceph-mon[25711]: Deploying cephadm binary to smithi130 2024-09-10T03:28:49.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:49 smithi164 ceph-mon[25711]: Deploying cephadm binary to smithi164 2024-09-10T03:28:50.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 47114 11.38 MB/sec warmup 119 sec latency 91.461 ms 2024-09-10T03:28:50.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:50 smithi130 ceph-mon[23614]: pgmap v4: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-10T03:28:50.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:50 smithi130 ceph-mon[23614]: mgrmap e25: smithi164.bkfwap(active, since 3s) 2024-09-10T03:28:50.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:50 smithi130 ceph-mon[23614]: [10/Sep/2024:03:28:49] ENGINE Bus STARTING 2024-09-10T03:28:50.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:50 smithi130 ceph-mon[23614]: [10/Sep/2024:03:28:49] ENGINE Serving on http://172.21.15.164:8765 2024-09-10T03:28:50.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:50 smithi130 ceph-mon[23614]: [10/Sep/2024:03:28:49] ENGINE Serving on https://172.21.15.164:7150 2024-09-10T03:28:50.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:50 smithi130 ceph-mon[23614]: [10/Sep/2024:03:28:49] ENGINE Bus STARTED 2024-09-10T03:28:50.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:50 smithi130 ceph-mon[23614]: [10/Sep/2024:03:28:49] ENGINE Error in HTTPServer.serve 2024-09-10T03:28:50.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Traceback (most recent call last): 2024-09-10T03:28:50.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-10T03:28:50.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: self._connections.run(self.expiration_interval) 2024-09-10T03:28:50.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-10T03:28:50.427 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: self._run(expiration_interval) 2024-09-10T03:28:50.427 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-10T03:28:50.427 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-10T03:28:50.427 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-10T03:28:50.427 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-10T03:28:50.427 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-10T03:28:50.427 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: s = self.context.wrap_socket( 2024-09-10T03:28:50.427 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-10T03:28:50.427 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: return self.sslsocket_class._create( 2024-09-10T03:28:50.427 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-10T03:28:50.427 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: self.do_handshake() 2024-09-10T03:28:50.427 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-10T03:28:50.427 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: self._sslobj.do_handshake() 2024-09-10T03:28:50.427 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-10T03:28:50.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:50 smithi164 ceph-mon[25711]: pgmap v4: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-10T03:28:50.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:50 smithi164 ceph-mon[25711]: mgrmap e25: smithi164.bkfwap(active, since 3s) 2024-09-10T03:28:50.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:50 smithi164 ceph-mon[25711]: [10/Sep/2024:03:28:49] ENGINE Bus STARTING 2024-09-10T03:28:50.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:50 smithi164 ceph-mon[25711]: [10/Sep/2024:03:28:49] ENGINE Serving on http://172.21.15.164:8765 2024-09-10T03:28:50.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:50 smithi164 ceph-mon[25711]: [10/Sep/2024:03:28:49] ENGINE Serving on https://172.21.15.164:7150 2024-09-10T03:28:50.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:50 smithi164 ceph-mon[25711]: [10/Sep/2024:03:28:49] ENGINE Bus STARTED 2024-09-10T03:28:50.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:50 smithi164 ceph-mon[25711]: [10/Sep/2024:03:28:49] ENGINE Error in HTTPServer.serve 2024-09-10T03:28:50.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Traceback (most recent call last): 2024-09-10T03:28:50.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-10T03:28:50.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: self._connections.run(self.expiration_interval) 2024-09-10T03:28:50.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-10T03:28:50.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: self._run(expiration_interval) 2024-09-10T03:28:50.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-10T03:28:50.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-10T03:28:50.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-10T03:28:50.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-10T03:28:50.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-10T03:28:50.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: s = self.context.wrap_socket( 2024-09-10T03:28:50.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-10T03:28:50.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: return self.sslsocket_class._create( 2024-09-10T03:28:50.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-10T03:28:50.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: self.do_handshake() 2024-09-10T03:28:50.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-10T03:28:50.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: self._sslobj.do_handshake() 2024-09-10T03:28:50.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-10T03:28:52.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 48064 10.37 MB/sec execute 1 sec latency 76.314 ms 2024-09-10T03:28:52.213 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:52 smithi130 ceph-mon[23614]: pgmap v5: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-10T03:28:52.214 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:52 smithi130 ceph-mon[23614]: mgrmap e26: smithi164.bkfwap(active, since 5s) 2024-09-10T03:28:52.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:52 smithi164 ceph-mon[25711]: pgmap v5: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-10T03:28:52.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:52 smithi164 ceph-mon[25711]: mgrmap e26: smithi164.bkfwap(active, since 5s) 2024-09-10T03:28:53.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 48623 12.16 MB/sec execute 2 sec latency 65.207 ms 2024-09-10T03:28:54.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 49133 14.93 MB/sec execute 3 sec latency 42.973 ms 2024-09-10T03:28:54.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:54 smithi130 ceph-mon[23614]: pgmap v6: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-10T03:28:54.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:54 smithi164 ceph-mon[25711]: pgmap v6: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-10T03:28:54.857 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:28:55.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 49574 16.24 MB/sec execute 4 sec latency 38.622 ms 2024-09-10T03:28:55.560 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:55 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:55.560 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:55 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:55.560 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:55 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:55.560 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:55 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:55.632 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:55 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:55.632 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:55 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:55.632 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:55 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:55.632 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:55 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:55.931 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:28:55.931 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (7m) 2m ago 8m 24.2M - ba2b418f427c a430c6fad8b8 2024-09-10T03:28:55.931 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (8m) 2m ago 8m 6400k - 17.2.0 e1d6a67b021e 138ed3222ba5 2024-09-10T03:28:55.931 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (8m) 1s ago 8m 6405k - 17.2.0 e1d6a67b021e 3623793da55d 2024-09-10T03:28:55.931 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (7m) 2m ago 8m 52.8M - 8.3.5 dad864ee21e9 1e4f86d07930 2024-09-10T03:28:55.931 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (2m) 2m ago 2m 15.0M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:28:55.931 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (2m) 1s ago 2m 14.1M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:28:55.931 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:9283 running (9m) 2m ago 9m 462M - 17.2.0 e1d6a67b021e 197f48917434 2024-09-10T03:28:55.932 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283 running (29s) 1s ago 8m 498M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e9559882d6ad 2024-09-10T03:28:55.932 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (9m) 2m ago 10m 46.5M 2048M 17.2.0 e1d6a67b021e f2b698da8d6e 2024-09-10T03:28:55.932 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (7m) 1s ago 7m 39.4M 2048M 17.2.0 e1d6a67b021e 7a928b0b045c 2024-09-10T03:28:55.932 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (2m) 2m ago 2m 50.0M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:28:55.932 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (2m) 1s ago 2m 75.5M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:28:55.932 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (8m) 2m ago 8m 17.4M - 1dbe0e931976 fcb1f0cd878a 2024-09-10T03:28:55.932 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (7m) 1s ago 7m 17.6M - 1.3.1 1dbe0e931976 1b5c522b94af 2024-09-10T03:28:55.932 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (7m) 2m ago 7m 43.2M 3216M 17.2.0 e1d6a67b021e 1d66043abecd 2024-09-10T03:28:55.932 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (6m) 2m ago 6m 42.9M 3216M 17.2.0 e1d6a67b021e ca1ec04aecfa 2024-09-10T03:28:55.932 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (6m) 2m ago 6m 41.0M 3216M 17.2.0 e1d6a67b021e 5883f5d5e969 2024-09-10T03:28:55.932 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (5m) 2m ago 5m 42.3M 3216M 17.2.0 e1d6a67b021e b32084e1a404 2024-09-10T03:28:55.933 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (5m) 1s ago 5m 215M 3984M 17.2.0 e1d6a67b021e 3b5c9f00f64d 2024-09-10T03:28:55.933 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (4m) 1s ago 4m 202M 3984M 17.2.0 e1d6a67b021e f16357ac331e 2024-09-10T03:28:55.933 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (4m) 1s ago 4m 199M 3984M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:28:55.933 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (4m) 1s ago 4m 293M 3984M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:28:55.933 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (7m) 2m ago 7m 62.8M - 514e6a882f6e a405bca101aa 2024-09-10T03:28:56.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 49920 14.99 MB/sec execute 5 sec latency 70.417 ms 2024-09-10T03:28:56.318 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:56 smithi130 ceph-mon[23614]: from='client.24459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:28:56.318 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:56 smithi130 ceph-mon[23614]: pgmap v7: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 3.5 MiB/s wr, 429 op/s 2024-09-10T03:28:56.318 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:56 smithi130 ceph-mon[23614]: from='client.24463 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:28:56.318 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:56 smithi130 ceph-mon[23614]: Standby manager daemon smithi130.axcmel started 2024-09-10T03:28:56.319 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:56 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.130:0/47993579' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi130.axcmel/crt"}]: dispatch 2024-09-10T03:28:56.319 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:56 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.130:0/47993579' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-10T03:28:56.319 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:56 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.130:0/47993579' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi130.axcmel/key"}]: dispatch 2024-09-10T03:28:56.319 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:56 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.130:0/47993579' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-10T03:28:56.562 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:28:56.562 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:28:56.562 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:28:56.562 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:28:56.562 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:28:56.562 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-10T03:28:56.562 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 1 2024-09-10T03:28:56.562 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:28:56.562 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:28:56.562 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-10T03:28:56.562 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:28:56.563 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:28:56.563 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:28:56.563 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:28:56.563 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:28:56.563 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-10T03:28:56.563 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 1 2024-09-10T03:28:56.563 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:28:56.563 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:28:56.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:56 smithi164 ceph-mon[25711]: from='client.24459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:28:56.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:56 smithi164 ceph-mon[25711]: pgmap v7: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 3.5 MiB/s wr, 429 op/s 2024-09-10T03:28:56.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:56 smithi164 ceph-mon[25711]: from='client.24463 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:28:56.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:56 smithi164 ceph-mon[25711]: Standby manager daemon smithi130.axcmel started 2024-09-10T03:28:56.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:56 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.130:0/47993579' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi130.axcmel/crt"}]: dispatch 2024-09-10T03:28:56.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:56 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.130:0/47993579' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-10T03:28:56.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:56 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.130:0/47993579' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi130.axcmel/key"}]: dispatch 2024-09-10T03:28:56.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:56 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.130:0/47993579' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-10T03:28:57.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 50244 13.48 MB/sec execute 6 sec latency 72.575 ms 2024-09-10T03:28:57.072 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:28:57.072 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:28:57.072 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:28:57.072 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:28:57.072 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-09-10T03:28:57.072 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "1/23 daemons upgraded", 2024-09-10T03:28:57.072 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "", 2024-09-10T03:28:57.072 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:28:57.073 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:28:57.362 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:57 smithi130 ceph-mon[23614]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:28:57.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:57 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mgr metadata", "who": "smithi130.axcmel", "id": "smithi130.axcmel"}]: dispatch 2024-09-10T03:28:57.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:57 smithi130 ceph-mon[23614]: mgrmap e27: smithi164.bkfwap(active, since 10s), standbys: smithi130.axcmel 2024-09-10T03:28:57.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:57 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/743209513' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-10T03:28:57.671 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-09-10T03:28:57.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:57 smithi164 ceph-mon[25711]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:28:57.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:57 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mgr metadata", "who": "smithi130.axcmel", "id": "smithi130.axcmel"}]: dispatch 2024-09-10T03:28:57.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:57 smithi164 ceph-mon[25711]: mgrmap e27: smithi164.bkfwap(active, since 10s), standbys: smithi130.axcmel 2024-09-10T03:28:57.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:57 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/743209513' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-10T03:28:58.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 50516 12.10 MB/sec execute 7 sec latency 64.416 ms 2024-09-10T03:28:58.619 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:58 smithi130 ceph-mon[23614]: pgmap v8: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 2.7 MiB/s wr, 333 op/s 2024-09-10T03:28:58.619 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:58 smithi130 ceph-mon[23614]: from='client.24473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:28:58.619 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:58 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:58.619 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:58 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:58.619 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:58 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/2258215009' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-10T03:28:58.619 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:58 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:58.619 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:58 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:58.620 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:58 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:58.620 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:58 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:58.620 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:58 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:58.620 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:58 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:58.620 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:58 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:58.620 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:58 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:58.620 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:58 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:58.620 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:58 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:58.620 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:58 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:58.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:58 smithi164 ceph-mon[25711]: pgmap v8: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 2.7 MiB/s wr, 333 op/s 2024-09-10T03:28:58.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:58 smithi164 ceph-mon[25711]: from='client.24473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:28:58.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:58 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:58.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:58 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:58.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:58 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/2258215009' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-10T03:28:58.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:58 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:58.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:58 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:58.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:58 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:58.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:58 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:58.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:58 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:58.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:58 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:58.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:58 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:58.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:58 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:58.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:58 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:58.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:58 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:58.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:58 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:59.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 50944 11.90 MB/sec execute 8 sec latency 33.247 ms 2024-09-10T03:28:59.637 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:59 smithi130 ceph-mon[23614]: Adjusting osd_memory_target on smithi130 to 1936M 2024-09-10T03:28:59.637 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:59 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:59.637 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:59 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:59.637 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:59 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:59.637 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:59 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:59.637 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:59 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:59.637 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:59 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:59.637 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:59 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:59.637 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:59 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:59.638 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:59 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:59.638 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:59 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:59.638 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:59 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:59.638 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:59 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:28:59.638 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:28:59 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:28:59.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:59 smithi164 ceph-mon[25711]: Adjusting osd_memory_target on smithi130 to 1936M 2024-09-10T03:28:59.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:59 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:59.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:59 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:59.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:59 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:59.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:59 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:59.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:59 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:59.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:59 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:59.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:59 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:59.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:59 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:59.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:59 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:59.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:59 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-10T03:28:59.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:59 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:28:59.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:59 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:28:59.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:28:59 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:29:00.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 51434 11.70 MB/sec execute 9 sec latency 95.581 ms 2024-09-10T03:29:00.646 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: Adjusting osd_memory_target on smithi164 to 2704M 2024-09-10T03:29:00.646 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: Updating smithi130:/etc/ceph/ceph.conf 2024-09-10T03:29:00.646 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: Updating smithi164:/etc/ceph/ceph.conf 2024-09-10T03:29:00.646 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: pgmap v9: 97 pgs: 97 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 5.3 MiB/s wr, 523 op/s 2024-09-10T03:29:00.646 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: Updating smithi164:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.conf 2024-09-10T03:29:00.646 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: Updating smithi130:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.conf 2024-09-10T03:29:00.646 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2024-09-10T03:29:00.647 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-09-10T03:29:00.647 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: Updating smithi164:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.client.admin.keyring 2024-09-10T03:29:00.647 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: Updating smithi130:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.client.admin.keyring 2024-09-10T03:29:00.647 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:00.647 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:00.647 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:00.647 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:00.647 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:00.648 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:00.648 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:00.648 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:00.648 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:00.648 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:00 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: Adjusting osd_memory_target on smithi164 to 2704M 2024-09-10T03:29:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: Updating smithi130:/etc/ceph/ceph.conf 2024-09-10T03:29:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: Updating smithi164:/etc/ceph/ceph.conf 2024-09-10T03:29:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: pgmap v9: 97 pgs: 97 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 5.3 MiB/s wr, 523 op/s 2024-09-10T03:29:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: Updating smithi164:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.conf 2024-09-10T03:29:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: Updating smithi130:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.conf 2024-09-10T03:29:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2024-09-10T03:29:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-09-10T03:29:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: Updating smithi164:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.client.admin.keyring 2024-09-10T03:29:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: Updating smithi130:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.client.admin.keyring 2024-09-10T03:29:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:00.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:00.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:00.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:00.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:00.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:00.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:00.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:00.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:00 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:01.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 52033 11.76 MB/sec execute 10 sec latency 41.930 ms 2024-09-10T03:29:01.665 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:01 smithi130 ceph-mon[23614]: Reconfiguring alertmanager.smithi130 (dependencies changed)... 2024-09-10T03:29:01.665 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:01 smithi130 ceph-mon[23614]: Deploying daemon alertmanager.smithi130 on smithi130 2024-09-10T03:29:01.859 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:01 smithi164 ceph-mon[25711]: Reconfiguring alertmanager.smithi130 (dependencies changed)... 2024-09-10T03:29:01.859 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:01 smithi164 ceph-mon[25711]: Deploying daemon alertmanager.smithi130 on smithi130 2024-09-10T03:29:02.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 52500 12.11 MB/sec execute 11 sec latency 68.090 ms 2024-09-10T03:29:02.692 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:02 smithi130 ceph-mon[23614]: pgmap v10: 97 pgs: 97 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 4.8 MiB/s wr, 474 op/s 2024-09-10T03:29:02.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:02 smithi164 ceph-mon[25711]: pgmap v10: 97 pgs: 97 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 4.8 MiB/s wr, 474 op/s 2024-09-10T03:29:03.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 52964 12.68 MB/sec execute 12 sec latency 35.624 ms 2024-09-10T03:29:03.715 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:03 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:29:03.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:03 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:29:04.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 53353 12.91 MB/sec execute 13 sec latency 48.681 ms 2024-09-10T03:29:04.722 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:04 smithi130 ceph-mon[23614]: pgmap v11: 97 pgs: 97 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 4.8 MiB/s wr, 474 op/s 2024-09-10T03:29:04.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:04 smithi164 ceph-mon[25711]: pgmap v11: 97 pgs: 97 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 4.8 MiB/s wr, 474 op/s 2024-09-10T03:29:05.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 53682 12.52 MB/sec execute 14 sec latency 59.752 ms 2024-09-10T03:29:06.011 INFO:teuthology.orchestra.run.smithi130.stdout: 5 54036 11.97 MB/sec execute 15 sec latency 54.039 ms 2024-09-10T03:29:06.760 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:06 smithi130 ceph-mon[23614]: pgmap v12: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 7.6 MiB/s wr, 754 op/s 2024-09-10T03:29:06.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:06 smithi164 ceph-mon[25711]: pgmap v12: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 7.6 MiB/s wr, 754 op/s 2024-09-10T03:29:07.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 54391 11.69 MB/sec execute 16 sec latency 53.115 ms 2024-09-10T03:29:08.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 54930 11.80 MB/sec execute 17 sec latency 31.452 ms 2024-09-10T03:29:08.695 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:08 smithi130 ceph-mon[23614]: pgmap v13: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.6 MiB/s wr, 507 op/s 2024-09-10T03:29:08.880 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:08 smithi164 ceph-mon[25711]: pgmap v13: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.6 MiB/s wr, 507 op/s 2024-09-10T03:29:09.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 55474 11.60 MB/sec execute 18 sec latency 29.270 ms 2024-09-10T03:29:10.015 INFO:teuthology.orchestra.run.smithi130.stdout: 5 55948 11.78 MB/sec execute 19 sec latency 37.978 ms 2024-09-10T03:29:10.742 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:10 smithi130 ceph-mon[23614]: pgmap v14: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 760 op/s 2024-09-10T03:29:10.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:10 smithi164 ceph-mon[25711]: pgmap v14: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 760 op/s 2024-09-10T03:29:11.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 56360 12.07 MB/sec execute 20 sec latency 37.541 ms 2024-09-10T03:29:12.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 56854 12.46 MB/sec execute 21 sec latency 42.666 ms 2024-09-10T03:29:12.771 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:12 smithi130 ceph-mon[23614]: pgmap v15: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.1 MiB/s wr, 532 op/s 2024-09-10T03:29:12.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:12 smithi164 ceph-mon[25711]: pgmap v15: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.1 MiB/s wr, 532 op/s 2024-09-10T03:29:13.010 INFO:teuthology.orchestra.run.smithi130.stdout: 5 57148 12.27 MB/sec execute 22 sec latency 54.274 ms 2024-09-10T03:29:14.011 INFO:teuthology.orchestra.run.smithi130.stdout: 5 57433 11.89 MB/sec execute 23 sec latency 69.996 ms 2024-09-10T03:29:14.793 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:14 smithi130 ceph-mon[23614]: pgmap v16: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.1 MiB/s wr, 532 op/s 2024-09-10T03:29:14.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:14 smithi164 ceph-mon[25711]: pgmap v16: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.1 MiB/s wr, 532 op/s 2024-09-10T03:29:15.011 INFO:teuthology.orchestra.run.smithi130.stdout: 5 57802 11.68 MB/sec execute 24 sec latency 56.546 ms 2024-09-10T03:29:16.011 INFO:teuthology.orchestra.run.smithi130.stdout: 5 58288 11.65 MB/sec execute 25 sec latency 41.501 ms 2024-09-10T03:29:16.806 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:16 smithi130 ceph-mon[23614]: pgmap v17: 97 pgs: 97 active+clean; 321 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 784 op/s 2024-09-10T03:29:16.807 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:16 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:16.807 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:16 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:16.807 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:16 smithi130 ceph-mon[23614]: Reconfiguring grafana.smithi130 (dependencies changed)... 2024-09-10T03:29:16.807 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:16 smithi130 ceph-mon[23614]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-10T03:29:17.011 INFO:teuthology.orchestra.run.smithi130.stdout: 5 58811 11.57 MB/sec execute 26 sec latency 60.779 ms 2024-09-10T03:29:17.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:16 smithi164 ceph-mon[25711]: pgmap v17: 97 pgs: 97 active+clean; 321 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 784 op/s 2024-09-10T03:29:17.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:16 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:17.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:16 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:17.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:16 smithi164 ceph-mon[25711]: Reconfiguring grafana.smithi130 (dependencies changed)... 2024-09-10T03:29:17.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:16 smithi164 ceph-mon[25711]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-10T03:29:18.011 INFO:teuthology.orchestra.run.smithi130.stdout: 5 59404 11.65 MB/sec execute 27 sec latency 45.061 ms 2024-09-10T03:29:18.063 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:17 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:18.063 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:17 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:18.063 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:17 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-10T03:29:18.063 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:17 smithi130 ceph-mon[23614]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-10T03:29:18.063 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:17 smithi130 ceph-mon[23614]: Reconfiguring daemon grafana.smithi130 on smithi130 2024-09-10T03:29:18.063 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:17 smithi130 ceph-mon[23614]: pgmap v18: 97 pgs: 97 active+clean; 321 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 504 op/s 2024-09-10T03:29:18.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:17 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:18.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:17 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:18.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:17 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-10T03:29:18.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:17 smithi164 ceph-mon[25711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-10T03:29:18.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:17 smithi164 ceph-mon[25711]: Reconfiguring daemon grafana.smithi130 on smithi130 2024-09-10T03:29:18.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:17 smithi164 ceph-mon[25711]: pgmap v18: 97 pgs: 97 active+clean; 321 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 504 op/s 2024-09-10T03:29:19.011 INFO:teuthology.orchestra.run.smithi130.stdout: 5 60090 12.28 MB/sec execute 28 sec latency 29.462 ms 2024-09-10T03:29:19.083 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:18 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:29:19.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:18 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:29:20.011 INFO:teuthology.orchestra.run.smithi130.stdout: 5 60494 12.43 MB/sec execute 29 sec latency 65.373 ms 2024-09-10T03:29:20.103 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:19 smithi130 ceph-mon[23614]: pgmap v19: 97 pgs: 97 active+clean; 321 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 782 op/s 2024-09-10T03:29:20.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:19 smithi164 ceph-mon[25711]: pgmap v19: 97 pgs: 97 active+clean; 321 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 782 op/s 2024-09-10T03:29:21.011 INFO:teuthology.orchestra.run.smithi130.stdout: 5 60831 12.24 MB/sec execute 30 sec latency 40.807 ms 2024-09-10T03:29:22.011 INFO:teuthology.orchestra.run.smithi130.stdout: 5 61168 11.97 MB/sec execute 31 sec latency 105.010 ms 2024-09-10T03:29:22.169 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:22 smithi130 ceph-mon[23614]: pgmap v20: 97 pgs: 97 active+clean; 321 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 529 op/s 2024-09-10T03:29:22.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:22 smithi164 ceph-mon[25711]: pgmap v20: 97 pgs: 97 active+clean; 321 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 529 op/s 2024-09-10T03:29:23.011 INFO:teuthology.orchestra.run.smithi130.stdout: 5 61496 11.83 MB/sec execute 32 sec latency 100.061 ms 2024-09-10T03:29:24.011 INFO:teuthology.orchestra.run.smithi130.stdout: 5 61959 11.83 MB/sec execute 33 sec latency 34.340 ms 2024-09-10T03:29:24.195 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:24 smithi130 ceph-mon[23614]: pgmap v21: 97 pgs: 97 active+clean; 321 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 529 op/s 2024-09-10T03:29:24.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:24 smithi164 ceph-mon[25711]: pgmap v21: 97 pgs: 97 active+clean; 321 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 529 op/s 2024-09-10T03:29:25.012 INFO:teuthology.orchestra.run.smithi130.stdout: 5 62436 11.71 MB/sec execute 34 sec latency 37.207 ms 2024-09-10T03:29:25.716 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:25 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:25.716 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:25 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:25.716 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:25 smithi130 ceph-mon[23614]: Reconfiguring node-exporter.smithi130 (dependencies changed)... 2024-09-10T03:29:25.716 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:25 smithi130 ceph-mon[23614]: Deploying daemon node-exporter.smithi130 on smithi130 2024-09-10T03:29:25.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:25 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:25.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:25 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:25.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:25 smithi164 ceph-mon[25711]: Reconfiguring node-exporter.smithi130 (dependencies changed)... 2024-09-10T03:29:25.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:25 smithi164 ceph-mon[25711]: Deploying daemon node-exporter.smithi130 on smithi130 2024-09-10T03:29:26.012 INFO:teuthology.orchestra.run.smithi130.stdout: 5 62895 11.75 MB/sec execute 35 sec latency 33.552 ms 2024-09-10T03:29:26.727 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:26 smithi130 ceph-mon[23614]: pgmap v22: 97 pgs: 97 active+clean; 337 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.4 MiB/s wr, 856 op/s 2024-09-10T03:29:26.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:26 smithi164 ceph-mon[25711]: pgmap v22: 97 pgs: 97 active+clean; 337 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.4 MiB/s wr, 856 op/s 2024-09-10T03:29:27.012 INFO:teuthology.orchestra.run.smithi130.stdout: 5 63386 11.97 MB/sec execute 36 sec latency 29.688 ms 2024-09-10T03:29:28.012 INFO:teuthology.orchestra.run.smithi130.stdout: 5 63816 12.12 MB/sec execute 37 sec latency 62.288 ms 2024-09-10T03:29:28.169 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:29:28.797 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:28 smithi130 ceph-mon[23614]: pgmap v23: 97 pgs: 97 active+clean; 337 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 604 op/s 2024-09-10T03:29:28.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:28 smithi164 ceph-mon[25711]: pgmap v23: 97 pgs: 97 active+clean; 337 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 604 op/s 2024-09-10T03:29:29.012 INFO:teuthology.orchestra.run.smithi130.stdout: 5 64075 12.01 MB/sec execute 38 sec latency 46.138 ms 2024-09-10T03:29:29.072 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:29:29.073 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 starting - - - - 2024-09-10T03:29:29.073 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (9m) 31s ago 9m 6400k - 17.2.0 e1d6a67b021e 138ed3222ba5 2024-09-10T03:29:29.073 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (8m) 34s ago 8m 6405k - 17.2.0 e1d6a67b021e 3623793da55d 2024-09-10T03:29:29.073 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 starting - - - - 2024-09-10T03:29:29.073 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (3m) 31s ago 3m 33.7M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:29:29.073 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (3m) 34s ago 3m 14.1M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:29:29.073 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:9283 running (10m) 31s ago 10m 193M - 17.2.0 e1d6a67b021e 197f48917434 2024-09-10T03:29:29.073 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283 running (62s) 34s ago 8m 498M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e9559882d6ad 2024-09-10T03:29:29.073 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (10m) 31s ago 10m 50.4M 2048M 17.2.0 e1d6a67b021e f2b698da8d6e 2024-09-10T03:29:29.073 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (8m) 34s ago 8m 39.4M 2048M 17.2.0 e1d6a67b021e 7a928b0b045c 2024-09-10T03:29:29.073 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (2m) 31s ago 2m 250M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:29:29.073 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (2m) 34s ago 2m 75.5M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:29:29.073 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (8m) 31s ago 8m 17.8M - 1.3.1 1dbe0e931976 fcb1f0cd878a 2024-09-10T03:29:29.074 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (8m) 34s ago 8m 17.6M - 1.3.1 1dbe0e931976 1b5c522b94af 2024-09-10T03:29:29.074 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (7m) 31s ago 7m 250M 1936M 17.2.0 e1d6a67b021e 1d66043abecd 2024-09-10T03:29:29.074 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (7m) 31s ago 7m 182M 1936M 17.2.0 e1d6a67b021e ca1ec04aecfa 2024-09-10T03:29:29.074 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (6m) 31s ago 6m 150M 1936M 17.2.0 e1d6a67b021e 5883f5d5e969 2024-09-10T03:29:29.074 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (6m) 31s ago 6m 194M 1936M 17.2.0 e1d6a67b021e b32084e1a404 2024-09-10T03:29:29.074 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (5m) 34s ago 5m 215M 2704M 17.2.0 e1d6a67b021e 3b5c9f00f64d 2024-09-10T03:29:29.074 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (5m) 34s ago 5m 202M 2704M 17.2.0 e1d6a67b021e f16357ac331e 2024-09-10T03:29:29.074 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (5m) 34s ago 5m 199M 2704M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:29:29.074 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (4m) 34s ago 4m 293M 2704M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:29:29.074 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (8m) 31s ago 8m 65.0M - 2.33.4 514e6a882f6e a405bca101aa 2024-09-10T03:29:29.669 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:29:29.669 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:29:29.669 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:29:29.669 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:29:29.669 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:29:29.669 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-10T03:29:29.669 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 1 2024-09-10T03:29:29.669 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:29:29.670 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:29:29.670 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-10T03:29:29.670 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:29:29.670 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:29:29.670 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:29:29.670 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:29:29.670 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:29:29.670 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-10T03:29:29.670 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 1 2024-09-10T03:29:29.670 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:29:29.671 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:29:29.814 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:29 smithi130 ceph-mon[23614]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:29:29.814 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:29 smithi130 ceph-mon[23614]: from='client.14740 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:29:29.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:29 smithi164 ceph-mon[25711]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:29:29.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:29 smithi164 ceph-mon[25711]: from='client.14740 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:29:30.012 INFO:teuthology.orchestra.run.smithi130.stdout: 5 64343 11.85 MB/sec execute 39 sec latency 71.279 ms 2024-09-10T03:29:30.131 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:29:30.132 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:29:30.132 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:29:30.132 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:29:30.132 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-09-10T03:29:30.132 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "1/23 daemons upgraded", 2024-09-10T03:29:30.132 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "", 2024-09-10T03:29:30.132 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:29:30.132 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:29:30.824 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:30 smithi130 ceph-mon[23614]: pgmap v24: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 865 op/s 2024-09-10T03:29:30.825 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:30 smithi130 ceph-mon[23614]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:29:30.825 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:30 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3399612103' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-10T03:29:30.834 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-09-10T03:29:30.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:30 smithi164 ceph-mon[25711]: pgmap v24: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 865 op/s 2024-09-10T03:29:30.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:30 smithi164 ceph-mon[25711]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:29:30.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:30 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/3399612103' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-10T03:29:31.012 INFO:teuthology.orchestra.run.smithi130.stdout: 5 64658 11.65 MB/sec execute 40 sec latency 72.392 ms 2024-09-10T03:29:31.838 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:31 smithi130 ceph-mon[23614]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:29:31.839 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:31 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/2048292672' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-10T03:29:31.858 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:31 smithi164 ceph-mon[25711]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:29:31.858 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:31 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/2048292672' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-10T03:29:32.012 INFO:teuthology.orchestra.run.smithi130.stdout: 5 65128 11.65 MB/sec execute 41 sec latency 28.645 ms 2024-09-10T03:29:32.664 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:32 smithi130 ceph-mon[23614]: pgmap v25: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 587 op/s 2024-09-10T03:29:32.664 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:32 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:32.664 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:32 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:32.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:32 smithi164 ceph-mon[25711]: pgmap v25: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 587 op/s 2024-09-10T03:29:32.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:32 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:32.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:32 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:33.016 INFO:teuthology.orchestra.run.smithi130.stdout: 5 65768 11.65 MB/sec execute 42 sec latency 115.943 ms 2024-09-10T03:29:33.679 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:33 smithi130 ceph-mon[23614]: Reconfiguring prometheus.smithi130 (dependencies changed)... 2024-09-10T03:29:33.680 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:33 smithi130 ceph-mon[23614]: Deploying daemon prometheus.smithi130 on smithi130 2024-09-10T03:29:33.680 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:33 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:29:33.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:33 smithi164 ceph-mon[25711]: Reconfiguring prometheus.smithi130 (dependencies changed)... 2024-09-10T03:29:33.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:33 smithi164 ceph-mon[25711]: Deploying daemon prometheus.smithi130 on smithi130 2024-09-10T03:29:33.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:33 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:29:34.016 INFO:teuthology.orchestra.run.smithi130.stdout: 5 66436 11.81 MB/sec execute 43 sec latency 26.314 ms 2024-09-10T03:29:34.683 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:34 smithi130 ceph-mon[23614]: pgmap v26: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 587 op/s 2024-09-10T03:29:34.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:34 smithi164 ceph-mon[25711]: pgmap v26: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 587 op/s 2024-09-10T03:29:35.016 INFO:teuthology.orchestra.run.smithi130.stdout: 5 67024 12.06 MB/sec execute 44 sec latency 28.340 ms 2024-09-10T03:29:36.016 INFO:teuthology.orchestra.run.smithi130.stdout: 5 67485 12.19 MB/sec execute 45 sec latency 39.175 ms 2024-09-10T03:29:36.704 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:36 smithi130 ceph-mon[23614]: pgmap v27: 97 pgs: 97 active+clean; 323 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 834 op/s 2024-09-10T03:29:36.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:36 smithi164 ceph-mon[25711]: pgmap v27: 97 pgs: 97 active+clean; 323 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 834 op/s 2024-09-10T03:29:37.016 INFO:teuthology.orchestra.run.smithi130.stdout: 5 67957 12.14 MB/sec execute 46 sec latency 32.678 ms 2024-09-10T03:29:38.016 INFO:teuthology.orchestra.run.smithi130.stdout: 5 68383 12.06 MB/sec execute 47 sec latency 29.565 ms 2024-09-10T03:29:38.719 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:38 smithi130 ceph-mon[23614]: pgmap v28: 97 pgs: 97 active+clean; 323 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 507 op/s 2024-09-10T03:29:38.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:38 smithi164 ceph-mon[25711]: pgmap v28: 97 pgs: 97 active+clean; 323 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 507 op/s 2024-09-10T03:29:39.017 INFO:teuthology.orchestra.run.smithi130.stdout: 5 68874 12.02 MB/sec execute 48 sec latency 33.104 ms 2024-09-10T03:29:40.017 INFO:teuthology.orchestra.run.smithi130.stdout: 5 69497 12.03 MB/sec execute 49 sec latency 53.783 ms 2024-09-10T03:29:40.774 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:40 smithi130 ceph-mon[23614]: pgmap v29: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.5 MiB/s wr, 816 op/s 2024-09-10T03:29:40.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:40 smithi164 ceph-mon[25711]: pgmap v29: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.5 MiB/s wr, 816 op/s 2024-09-10T03:29:41.017 INFO:teuthology.orchestra.run.smithi130.stdout: 5 70288 12.28 MB/sec execute 50 sec latency 59.958 ms 2024-09-10T03:29:42.017 INFO:teuthology.orchestra.run.smithi130.stdout: 5 70930 12.55 MB/sec execute 51 sec latency 25.026 ms 2024-09-10T03:29:42.800 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:42 smithi130 ceph-mon[23614]: pgmap v30: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 555 op/s 2024-09-10T03:29:42.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:42 smithi164 ceph-mon[25711]: pgmap v30: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 555 op/s 2024-09-10T03:29:43.017 INFO:teuthology.orchestra.run.smithi130.stdout: 5 71327 12.49 MB/sec execute 52 sec latency 40.895 ms 2024-09-10T03:29:44.017 INFO:teuthology.orchestra.run.smithi130.stdout: 5 71665 12.39 MB/sec execute 53 sec latency 39.298 ms 2024-09-10T03:29:44.843 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:44 smithi130 ceph-mon[23614]: pgmap v31: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 555 op/s 2024-09-10T03:29:44.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:44 smithi164 ceph-mon[25711]: pgmap v31: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 555 op/s 2024-09-10T03:29:45.017 INFO:teuthology.orchestra.run.smithi130.stdout: 5 72039 12.35 MB/sec execute 54 sec latency 40.589 ms 2024-09-10T03:29:46.017 INFO:teuthology.orchestra.run.smithi130.stdout: 5 72502 12.24 MB/sec execute 55 sec latency 48.970 ms 2024-09-10T03:29:46.629 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:46 smithi130 ceph-mon[23614]: pgmap v32: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.3 MiB/s wr, 890 op/s 2024-09-10T03:29:46.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:46 smithi164 ceph-mon[25711]: pgmap v32: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.3 MiB/s wr, 890 op/s 2024-09-10T03:29:47.017 INFO:teuthology.orchestra.run.smithi130.stdout: 5 73269 12.34 MB/sec execute 56 sec latency 61.124 ms 2024-09-10T03:29:48.017 INFO:teuthology.orchestra.run.smithi130.stdout: 5 73813 12.41 MB/sec execute 57 sec latency 29.838 ms 2024-09-10T03:29:48.647 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:48 smithi130 ceph-mon[23614]: pgmap v33: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.1 MiB/s wr, 643 op/s 2024-09-10T03:29:48.647 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:48 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:29:48.961 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:48 smithi164 ceph-mon[25711]: pgmap v33: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.1 MiB/s wr, 643 op/s 2024-09-10T03:29:48.962 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:48 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:29:49.017 INFO:teuthology.orchestra.run.smithi130.stdout: 5 74290 12.53 MB/sec execute 58 sec latency 30.586 ms 2024-09-10T03:29:50.017 INFO:teuthology.orchestra.run.smithi130.stdout: 5 74712 12.58 MB/sec execute 59 sec latency 61.074 ms 2024-09-10T03:29:50.658 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:50 smithi130 ceph-mon[23614]: pgmap v34: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.8 MiB/s wr, 935 op/s 2024-09-10T03:29:50.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:50 smithi164 ceph-mon[25711]: pgmap v34: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.8 MiB/s wr, 935 op/s 2024-09-10T03:29:51.017 INFO:teuthology.orchestra.run.smithi130.stdout: 5 75083 12.51 MB/sec execute 60 sec latency 68.050 ms 2024-09-10T03:29:52.017 INFO:teuthology.orchestra.run.smithi130.stdout: 5 75462 12.38 MB/sec execute 61 sec latency 43.058 ms 2024-09-10T03:29:52.678 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:52 smithi130 ceph-mon[23614]: pgmap v35: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.3 MiB/s wr, 626 op/s 2024-09-10T03:29:52.963 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:52 smithi164 ceph-mon[25711]: pgmap v35: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.3 MiB/s wr, 626 op/s 2024-09-10T03:29:53.022 INFO:teuthology.orchestra.run.smithi130.stdout: 5 76012 12.40 MB/sec execute 62 sec latency 33.923 ms 2024-09-10T03:29:54.022 INFO:teuthology.orchestra.run.smithi130.stdout: 5 76615 12.39 MB/sec execute 63 sec latency 37.087 ms 2024-09-10T03:29:54.721 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:54 smithi130 ceph-mon[23614]: pgmap v36: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.3 MiB/s wr, 626 op/s 2024-09-10T03:29:54.721 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:54 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:54.721 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:54 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:54.896 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:54 smithi164 ceph-mon[25711]: pgmap v36: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.3 MiB/s wr, 626 op/s 2024-09-10T03:29:54.897 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:54 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:54.897 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:54 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:29:55.032 INFO:teuthology.orchestra.run.smithi130.stdout: 5 77243 12.45 MB/sec execute 64 sec latency 25.130 ms 2024-09-10T03:29:55.734 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:55 smithi130 ceph-mon[23614]: Reconfiguring node-exporter.smithi164 (dependencies changed)... 2024-09-10T03:29:55.734 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:55 smithi130 ceph-mon[23614]: Deploying daemon node-exporter.smithi164 on smithi164 2024-09-10T03:29:55.948 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:55 smithi164 ceph-mon[25711]: Reconfiguring node-exporter.smithi164 (dependencies changed)... 2024-09-10T03:29:55.948 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:55 smithi164 ceph-mon[25711]: Deploying daemon node-exporter.smithi164 on smithi164 2024-09-10T03:29:56.032 INFO:teuthology.orchestra.run.smithi130.stdout: 5 77838 12.64 MB/sec execute 65 sec latency 28.717 ms 2024-09-10T03:29:56.738 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:56 smithi130 ceph-mon[23614]: pgmap v37: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.1 MiB/s wr, 904 op/s 2024-09-10T03:29:56.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:56 smithi164 ceph-mon[25711]: pgmap v37: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.1 MiB/s wr, 904 op/s 2024-09-10T03:29:57.032 INFO:teuthology.orchestra.run.smithi130.stdout: 5 78307 12.71 MB/sec execute 66 sec latency 29.952 ms 2024-09-10T03:29:58.032 INFO:teuthology.orchestra.run.smithi130.stdout: 5 78896 12.68 MB/sec execute 67 sec latency 24.316 ms 2024-09-10T03:29:58.771 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:29:58 smithi130 ceph-mon[23614]: pgmap v38: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.5 MiB/s wr, 569 op/s 2024-09-10T03:29:58.879 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:29:58 smithi164 ceph-mon[25711]: pgmap v38: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.5 MiB/s wr, 569 op/s 2024-09-10T03:29:59.033 INFO:teuthology.orchestra.run.smithi130.stdout: 5 79335 12.63 MB/sec execute 68 sec latency 53.323 ms 2024-09-10T03:30:00.033 INFO:teuthology.orchestra.run.smithi130.stdout: 5 79970 12.64 MB/sec execute 69 sec latency 46.637 ms 2024-09-10T03:30:00.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:00.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: pgmap v39: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 914 op/s 2024-09-10T03:30:00.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:00.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-10T03:30:00.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-10T03:30:00.281 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130.front.sepia.ceph.com:9093"}]: dispatch 2024-09-10T03:30:00.281 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130.front.sepia.ceph.com:9093"}]: dispatch 2024-09-10T03:30:00.281 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:00.281 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-10T03:30:00.281 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-10T03:30:00.281 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi130.front.sepia.ceph.com:3000"}]: dispatch 2024-09-10T03:30:00.281 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi130.front.sepia.ceph.com:3000"}]: dispatch 2024-09-10T03:30:00.281 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:00.281 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-10T03:30:00.281 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-10T03:30:00.281 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi130.front.sepia.ceph.com:9095"}]: dispatch 2024-09-10T03:30:00.281 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi130.front.sepia.ceph.com:9095"}]: dispatch 2024-09-10T03:30:00.281 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:00.281 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:30:00.282 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: Upgrade: Need to upgrade myself (mgr.smithi164.bkfwap) 2024-09-10T03:30:00.282 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: Upgrade: Need to upgrade myself (mgr.smithi164.bkfwap) 2024-09-10T03:30:00.282 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:30:00.282 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:30:00.282 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:00 smithi130 ceph-mon[23614]: overall HEALTH_OK 2024-09-10T03:30:00.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:00.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: pgmap v39: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 914 op/s 2024-09-10T03:30:00.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:00.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-10T03:30:00.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-10T03:30:00.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130.front.sepia.ceph.com:9093"}]: dispatch 2024-09-10T03:30:00.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130.front.sepia.ceph.com:9093"}]: dispatch 2024-09-10T03:30:00.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:00.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-10T03:30:00.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-10T03:30:00.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi130.front.sepia.ceph.com:3000"}]: dispatch 2024-09-10T03:30:00.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi130.front.sepia.ceph.com:3000"}]: dispatch 2024-09-10T03:30:00.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:00.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-10T03:30:00.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-10T03:30:00.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi130.front.sepia.ceph.com:9095"}]: dispatch 2024-09-10T03:30:00.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi130.front.sepia.ceph.com:9095"}]: dispatch 2024-09-10T03:30:00.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:00.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:30:00.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: Upgrade: Need to upgrade myself (mgr.smithi164.bkfwap) 2024-09-10T03:30:00.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: Upgrade: Need to upgrade myself (mgr.smithi164.bkfwap) 2024-09-10T03:30:00.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:30:00.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:30:00.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:00 smithi164 ceph-mon[25711]: overall HEALTH_OK 2024-09-10T03:30:01.033 INFO:teuthology.orchestra.run.smithi130.stdout: 5 80539 12.68 MB/sec execute 70 sec latency 26.271 ms 2024-09-10T03:30:01.283 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:01 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:30:01.284 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:01 smithi130 ceph-mon[23614]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:30:01.284 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:01 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:30:01.284 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:01 smithi130 ceph-mon[23614]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:30:01.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:01 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:30:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:01 smithi164 ceph-mon[25711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:30:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:01 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:30:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:01 smithi164 ceph-mon[25711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:30:01.467 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:30:02.033 INFO:teuthology.orchestra.run.smithi130.stdout: 5 81305 12.89 MB/sec execute 71 sec latency 23.384 ms 2024-09-10T03:30:02.288 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:02 smithi130 ceph-mon[23614]: pgmap v40: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.6 MiB/s wr, 622 op/s 2024-09-10T03:30:02.288 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:02 smithi130 ceph-mon[23614]: from='client.14758 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:30:02.288 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:02 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:02.288 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:02 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi130.axcmel", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-10T03:30:02.288 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:02 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi130.axcmel", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-10T03:30:02.288 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:02 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-10T03:30:02.288 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:02 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:30:02.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:02 smithi164 ceph-mon[25711]: pgmap v40: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.6 MiB/s wr, 622 op/s 2024-09-10T03:30:02.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:02 smithi164 ceph-mon[25711]: from='client.14758 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:30:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:02 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:02 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi130.axcmel", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-10T03:30:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:02 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi130.axcmel", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-10T03:30:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:02 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-10T03:30:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:02 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:30:02.642 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:30:02.642 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 starting - - - - 2024-09-10T03:30:02.643 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (9m) 65s ago 9m 6400k - 17.2.0 e1d6a67b021e 138ed3222ba5 2024-09-10T03:30:02.643 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (9m) 68s ago 9m 6405k - 17.2.0 e1d6a67b021e 3623793da55d 2024-09-10T03:30:02.643 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 starting - - - - 2024-09-10T03:30:02.643 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (3m) 65s ago 3m 33.7M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:30:02.643 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (3m) 68s ago 3m 14.1M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:30:02.643 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:9283 running (10m) 65s ago 10m 193M - 17.2.0 e1d6a67b021e 197f48917434 2024-09-10T03:30:02.643 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283 running (96s) 68s ago 9m 498M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e9559882d6ad 2024-09-10T03:30:02.643 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (10m) 65s ago 11m 50.4M 2048M 17.2.0 e1d6a67b021e f2b698da8d6e 2024-09-10T03:30:02.643 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (9m) 68s ago 9m 39.4M 2048M 17.2.0 e1d6a67b021e 7a928b0b045c 2024-09-10T03:30:02.643 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (3m) 65s ago 3m 250M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:30:02.643 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (3m) 68s ago 3m 75.5M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:30:02.643 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 starting - - - - 2024-09-10T03:30:02.643 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 starting - - - - 2024-09-10T03:30:02.643 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (8m) 65s ago 8m 250M 1936M 17.2.0 e1d6a67b021e 1d66043abecd 2024-09-10T03:30:02.644 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (7m) 65s ago 7m 182M 1936M 17.2.0 e1d6a67b021e ca1ec04aecfa 2024-09-10T03:30:02.644 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (7m) 65s ago 7m 150M 1936M 17.2.0 e1d6a67b021e 5883f5d5e969 2024-09-10T03:30:02.644 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (6m) 65s ago 6m 194M 1936M 17.2.0 e1d6a67b021e b32084e1a404 2024-09-10T03:30:02.644 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (6m) 68s ago 6m 215M 2704M 17.2.0 e1d6a67b021e 3b5c9f00f64d 2024-09-10T03:30:02.644 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (6m) 68s ago 6m 202M 2704M 17.2.0 e1d6a67b021e f16357ac331e 2024-09-10T03:30:02.644 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (5m) 68s ago 5m 199M 2704M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:30:02.644 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (5m) 68s ago 5m 293M 2704M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:30:02.644 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 starting - - - - 2024-09-10T03:30:03.042 INFO:teuthology.orchestra.run.smithi130.stdout: 5 81951 13.00 MB/sec execute 72 sec latency 33.426 ms 2024-09-10T03:30:03.303 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:03 smithi130 ceph-mon[23614]: Upgrade: Updating mgr.smithi130.axcmel 2024-09-10T03:30:03.303 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:03 smithi130 ceph-mon[23614]: Deploying daemon mgr.smithi130.axcmel on smithi130 2024-09-10T03:30:03.303 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:03 smithi130 ceph-mon[23614]: from='client.24501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:30:03.303 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:03 smithi130 ceph-mon[23614]: from='client.24505 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:30:03.373 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:30:03.373 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:30:03.373 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:30:03.374 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:30:03.374 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:30:03.374 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-10T03:30:03.374 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 1 2024-09-10T03:30:03.374 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:30:03.374 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:30:03.374 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-10T03:30:03.374 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:30:03.374 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:30:03.374 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:30:03.374 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:30:03.374 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:30:03.374 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-10T03:30:03.375 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 1 2024-09-10T03:30:03.375 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:30:03.375 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:30:03.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:03 smithi164 ceph-mon[25711]: Upgrade: Updating mgr.smithi130.axcmel 2024-09-10T03:30:03.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:03 smithi164 ceph-mon[25711]: Deploying daemon mgr.smithi130.axcmel on smithi130 2024-09-10T03:30:03.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:03 smithi164 ceph-mon[25711]: from='client.24501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:30:03.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:03 smithi164 ceph-mon[25711]: from='client.24505 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:30:03.985 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:30:03.985 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:30:03.985 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:30:03.985 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:30:03.985 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-09-10T03:30:03.985 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "1/23 daemons upgraded", 2024-09-10T03:30:03.986 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading mgr daemons", 2024-09-10T03:30:03.986 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:30:03.986 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:30:04.045 INFO:teuthology.orchestra.run.smithi130.stdout: 5 82491 12.95 MB/sec execute 73 sec latency 25.013 ms 2024-09-10T03:30:04.231 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:04 smithi130 ceph-mon[23614]: pgmap v41: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.6 MiB/s wr, 622 op/s 2024-09-10T03:30:04.231 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:04 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:30:04.231 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:04 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/2269837822' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-10T03:30:04.454 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:04 smithi164 ceph-mon[25711]: pgmap v41: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.6 MiB/s wr, 622 op/s 2024-09-10T03:30:04.454 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:04 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:30:04.454 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:04 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/2269837822' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-10T03:30:04.755 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-09-10T03:30:05.045 INFO:teuthology.orchestra.run.smithi130.stdout: 5 83064 12.96 MB/sec execute 74 sec latency 40.219 ms 2024-09-10T03:30:05.322 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:05 smithi130 ceph-mon[23614]: from='client.24513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:30:05.322 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:05 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/1488896542' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-10T03:30:05.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:05 smithi164 ceph-mon[25711]: from='client.24513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:30:05.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:05 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/1488896542' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-10T03:30:06.045 INFO:teuthology.orchestra.run.smithi130.stdout: 5 83531 12.90 MB/sec execute 75 sec latency 42.804 ms 2024-09-10T03:30:06.336 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:06 smithi130 ceph-mon[23614]: pgmap v42: 97 pgs: 97 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 968 op/s 2024-09-10T03:30:06.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:06 smithi164 ceph-mon[25711]: pgmap v42: 97 pgs: 97 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 968 op/s 2024-09-10T03:30:07.045 INFO:teuthology.orchestra.run.smithi130.stdout: 5 84070 12.93 MB/sec execute 76 sec latency 45.750 ms 2024-09-10T03:30:08.045 INFO:teuthology.orchestra.run.smithi130.stdout: 5 84536 12.94 MB/sec execute 77 sec latency 38.905 ms 2024-09-10T03:30:08.387 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:08 smithi130 ceph-mon[23614]: pgmap v43: 97 pgs: 97 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.5 MiB/s wr, 690 op/s 2024-09-10T03:30:08.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:08 smithi164 ceph-mon[25711]: pgmap v43: 97 pgs: 97 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.5 MiB/s wr, 690 op/s 2024-09-10T03:30:09.045 INFO:teuthology.orchestra.run.smithi130.stdout: 5 84952 13.00 MB/sec execute 78 sec latency 29.664 ms 2024-09-10T03:30:10.045 INFO:teuthology.orchestra.run.smithi130.stdout: 5 85279 13.01 MB/sec execute 79 sec latency 52.892 ms 2024-09-10T03:30:10.664 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:10 smithi130 ceph-mon[23614]: pgmap v44: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 999 op/s 2024-09-10T03:30:10.665 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:10 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:10.665 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:10 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:10.665 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:10 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:30:10.665 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:10 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:10.665 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:10 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:10.752 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:10 smithi164 ceph-mon[25711]: pgmap v44: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 999 op/s 2024-09-10T03:30:10.753 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:10 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:10.753 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:10 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:10.753 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:10 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:30:10.753 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:10 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:10.753 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:10 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:11.049 INFO:teuthology.orchestra.run.smithi130.stdout: 5 85590 12.90 MB/sec execute 80 sec latency 40.243 ms 2024-09-10T03:30:12.045 INFO:teuthology.orchestra.run.smithi130.stdout: 5 86003 12.85 MB/sec execute 81 sec latency 35.173 ms 2024-09-10T03:30:12.677 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:12 smithi130 ceph-mon[23614]: pgmap v45: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 654 op/s 2024-09-10T03:30:12.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:12 smithi164 ceph-mon[25711]: pgmap v45: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 654 op/s 2024-09-10T03:30:13.045 INFO:teuthology.orchestra.run.smithi130.stdout: 5 86423 12.82 MB/sec execute 82 sec latency 53.922 ms 2024-09-10T03:30:14.046 INFO:teuthology.orchestra.run.smithi130.stdout: 5 87027 12.79 MB/sec execute 83 sec latency 31.770 ms 2024-09-10T03:30:14.690 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:14 smithi130 ceph-mon[23614]: pgmap v46: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 654 op/s 2024-09-10T03:30:14.690 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:14 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:14.690 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:14 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:14.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:14 smithi164 ceph-mon[25711]: pgmap v46: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 654 op/s 2024-09-10T03:30:14.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:14 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:14.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:14 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:15.045 INFO:teuthology.orchestra.run.smithi130.stdout: 5 87598 12.84 MB/sec execute 84 sec latency 29.641 ms 2024-09-10T03:30:16.045 INFO:teuthology.orchestra.run.smithi130.stdout: 5 88448 13.03 MB/sec execute 85 sec latency 28.881 ms 2024-09-10T03:30:16.596 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:16 smithi130 ceph-mon[23614]: pgmap v47: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 893 op/s 2024-09-10T03:30:16.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:16 smithi164 ceph-mon[25711]: pgmap v47: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 893 op/s 2024-09-10T03:30:17.045 INFO:teuthology.orchestra.run.smithi130.stdout: 5 88963 13.08 MB/sec execute 86 sec latency 21.328 ms 2024-09-10T03:30:17.599 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:17 smithi130 ceph-mon[23614]: Standby manager daemon smithi130.axcmel restarted 2024-09-10T03:30:17.600 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:17 smithi130 ceph-mon[23614]: Standby manager daemon smithi130.axcmel started 2024-09-10T03:30:17.600 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:17 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi130.axcmel/crt"}]: dispatch 2024-09-10T03:30:17.600 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:17 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-10T03:30:17.600 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:17 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi130.axcmel/key"}]: dispatch 2024-09-10T03:30:17.600 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:17 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-10T03:30:17.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:17 smithi164 ceph-mon[25711]: Standby manager daemon smithi130.axcmel restarted 2024-09-10T03:30:17.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:17 smithi164 ceph-mon[25711]: Standby manager daemon smithi130.axcmel started 2024-09-10T03:30:17.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:17 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi130.axcmel/crt"}]: dispatch 2024-09-10T03:30:17.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:17 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-10T03:30:17.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:17 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi130.axcmel/key"}]: dispatch 2024-09-10T03:30:17.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:17 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-10T03:30:18.045 INFO:teuthology.orchestra.run.smithi130.stdout: 5 89451 13.03 MB/sec execute 87 sec latency 31.806 ms 2024-09-10T03:30:18.605 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:18 smithi130 ceph-mon[23614]: pgmap v48: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.4 MiB/s wr, 547 op/s 2024-09-10T03:30:18.605 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:18 smithi130 ceph-mon[23614]: mgrmap e28: smithi164.bkfwap(active, since 91s), standbys: smithi130.axcmel 2024-09-10T03:30:18.606 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:18 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:30:18.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:18 smithi164 ceph-mon[25711]: pgmap v48: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.4 MiB/s wr, 547 op/s 2024-09-10T03:30:18.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:18 smithi164 ceph-mon[25711]: mgrmap e28: smithi164.bkfwap(active, since 91s), standbys: smithi130.axcmel 2024-09-10T03:30:18.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:18 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:30:19.045 INFO:teuthology.orchestra.run.smithi130.stdout: 5 89959 13.03 MB/sec execute 88 sec latency 31.113 ms 2024-09-10T03:30:20.046 INFO:teuthology.orchestra.run.smithi130.stdout: 5 90483 12.98 MB/sec execute 89 sec latency 36.950 ms 2024-09-10T03:30:20.398 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:20 smithi130 ceph-mon[23614]: pgmap v49: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.2 MiB/s wr, 897 op/s 2024-09-10T03:30:20.398 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:20 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:20.398 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:20 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:20.398 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:20 smithi130 ceph-mon[23614]: mgrmap e29: smithi164.bkfwap(active, since 93s), standbys: smithi130.axcmel 2024-09-10T03:30:20.398 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:20 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:20.398 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:20 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:20.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:20 smithi164 ceph-mon[25711]: pgmap v49: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.2 MiB/s wr, 897 op/s 2024-09-10T03:30:20.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:20 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:20.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:20 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:20.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:20 smithi164 ceph-mon[25711]: mgrmap e29: smithi164.bkfwap(active, since 93s), standbys: smithi130.axcmel 2024-09-10T03:30:20.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:20 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:20.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:20 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:21.045 INFO:teuthology.orchestra.run.smithi130.stdout: 5 90825 12.90 MB/sec execute 90 sec latency 46.377 ms 2024-09-10T03:30:22.046 INFO:teuthology.orchestra.run.smithi130.stdout: 5 91278 12.98 MB/sec execute 91 sec latency 45.773 ms 2024-09-10T03:30:22.470 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:22 smithi130 ceph-mon[23614]: pgmap v50: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.2 MiB/s wr, 588 op/s 2024-09-10T03:30:22.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:22 smithi164 ceph-mon[25711]: pgmap v50: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.2 MiB/s wr, 588 op/s 2024-09-10T03:30:23.045 INFO:teuthology.orchestra.run.smithi130.stdout: 5 91761 12.99 MB/sec execute 92 sec latency 39.450 ms 2024-09-10T03:30:24.046 INFO:teuthology.orchestra.run.smithi130.stdout: 5 92123 13.00 MB/sec execute 93 sec latency 48.767 ms 2024-09-10T03:30:24.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:24 smithi130 ceph-mon[23614]: pgmap v51: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.2 MiB/s wr, 588 op/s 2024-09-10T03:30:24.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:24 smithi164 ceph-mon[25711]: pgmap v51: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.2 MiB/s wr, 588 op/s 2024-09-10T03:30:25.046 INFO:teuthology.orchestra.run.smithi130.stdout: 5 92425 12.98 MB/sec execute 94 sec latency 78.502 ms 2024-09-10T03:30:26.046 INFO:teuthology.orchestra.run.smithi130.stdout: 5 92837 12.92 MB/sec execute 95 sec latency 48.941 ms 2024-09-10T03:30:26.548 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:26 smithi130 ceph-mon[23614]: pgmap v52: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 855 op/s 2024-09-10T03:30:26.548 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:26 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:26.548 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:26 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:26.548 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:26 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:30:26.549 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:26 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:30:26.549 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:26 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:26.549 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:26 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:26.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:26 smithi164 ceph-mon[25711]: pgmap v52: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 855 op/s 2024-09-10T03:30:26.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:26 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:26.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:26 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:26.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:26 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:30:26.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:26 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:30:26.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:26 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:26.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:26 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' 2024-09-10T03:30:27.046 INFO:teuthology.orchestra.run.smithi130.stdout: 5 93146 12.84 MB/sec execute 96 sec latency 53.647 ms 2024-09-10T03:30:27.565 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:27 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:30:27.565 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:27 smithi130 ceph-mon[23614]: Upgrade: Need to upgrade myself (mgr.smithi164.bkfwap) 2024-09-10T03:30:27.565 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:27 smithi130 ceph-mon[23614]: Upgrade: Need to upgrade myself (mgr.smithi164.bkfwap) 2024-09-10T03:30:27.565 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:27 smithi130 ceph-mon[23614]: Failing over to other MGR 2024-09-10T03:30:27.565 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:27 smithi130 ceph-mon[23614]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mgr fail", "who": "smithi164.bkfwap"}]: dispatch 2024-09-10T03:30:27.565 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:27 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mgr fail", "who": "smithi164.bkfwap"}]: dispatch 2024-09-10T03:30:27.565 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:27 smithi130 ceph-mon[23614]: osdmap e72: 8 total, 8 up, 8 in 2024-09-10T03:30:27.565 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:27 smithi130 ceph-mon[23614]: Standby manager daemon smithi164.bkfwap started 2024-09-10T03:30:27.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:27 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:30:27.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:27 smithi164 ceph-mon[25711]: Upgrade: Need to upgrade myself (mgr.smithi164.bkfwap) 2024-09-10T03:30:27.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:27 smithi164 ceph-mon[25711]: Upgrade: Need to upgrade myself (mgr.smithi164.bkfwap) 2024-09-10T03:30:27.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:27 smithi164 ceph-mon[25711]: Failing over to other MGR 2024-09-10T03:30:27.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:27 smithi164 ceph-mon[25711]: from='mgr.24429 172.21.15.164:0/3959253578' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mgr fail", "who": "smithi164.bkfwap"}]: dispatch 2024-09-10T03:30:27.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:27 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "mgr fail", "who": "smithi164.bkfwap"}]: dispatch 2024-09-10T03:30:27.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:27 smithi164 ceph-mon[25711]: osdmap e72: 8 total, 8 up, 8 in 2024-09-10T03:30:27.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:27 smithi164 ceph-mon[25711]: Standby manager daemon smithi164.bkfwap started 2024-09-10T03:30:28.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 93460 12.80 MB/sec execute 97 sec latency 60.890 ms 2024-09-10T03:30:28.580 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd='[{"prefix": "mgr fail", "who": "smithi164.bkfwap"}]': finished 2024-09-10T03:30:28.580 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: mgrmap e30: smithi130.axcmel(active, starting, since 0.695698s), standbys: smithi164.bkfwap 2024-09-10T03:30:28.581 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi130"}]: dispatch 2024-09-10T03:30:28.581 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:30:28.581 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi130.ukcxnb"}]: dispatch 2024-09-10T03:30:28.581 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi164.awiezo"}]: dispatch 2024-09-10T03:30:28.581 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr metadata", "who": "smithi130.axcmel", "id": "smithi130.axcmel"}]: dispatch 2024-09-10T03:30:28.581 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr metadata", "who": "smithi164.bkfwap", "id": "smithi164.bkfwap"}]: dispatch 2024-09-10T03:30:28.581 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-10T03:30:28.581 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-10T03:30:28.581 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-10T03:30:28.582 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-10T03:30:28.582 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-10T03:30:28.582 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-10T03:30:28.582 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-10T03:30:28.582 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-10T03:30:28.582 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-10T03:30:28.582 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-10T03:30:28.583 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-10T03:30:28.583 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: Manager daemon smithi130.axcmel is now available 2024-09-10T03:30:28.583 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:30:28.583 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:30:28.583 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:30:28.583 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:30:28.583 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:30:28.583 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:30:28.584 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:28 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:30:28.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.24429 ' entity='mgr.smithi164.bkfwap' cmd='[{"prefix": "mgr fail", "who": "smithi164.bkfwap"}]': finished 2024-09-10T03:30:28.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: mgrmap e30: smithi130.axcmel(active, starting, since 0.695698s), standbys: smithi164.bkfwap 2024-09-10T03:30:28.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi130"}]: dispatch 2024-09-10T03:30:28.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata", "id": "smithi164"}]: dispatch 2024-09-10T03:30:28.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi130.ukcxnb"}]: dispatch 2024-09-10T03:30:28.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi164.awiezo"}]: dispatch 2024-09-10T03:30:28.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr metadata", "who": "smithi130.axcmel", "id": "smithi130.axcmel"}]: dispatch 2024-09-10T03:30:28.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr metadata", "who": "smithi164.bkfwap", "id": "smithi164.bkfwap"}]: dispatch 2024-09-10T03:30:28.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-10T03:30:28.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-10T03:30:28.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-10T03:30:28.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-10T03:30:28.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-10T03:30:28.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-10T03:30:28.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-10T03:30:28.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-10T03:30:28.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-10T03:30:28.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-10T03:30:28.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-10T03:30:28.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: Manager daemon smithi130.axcmel is now available 2024-09-10T03:30:28.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:30:28.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:30:28.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:30:28.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:30:28.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"}]: dispatch 2024-09-10T03:30:28.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:30:28.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:28 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"}]: dispatch 2024-09-10T03:30:29.046 INFO:teuthology.orchestra.run.smithi130.stdout: 5 93838 12.75 MB/sec execute 98 sec latency 66.300 ms 2024-09-10T03:30:29.846 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:29 smithi130 ceph-mon[23614]: mgrmap e31: smithi130.axcmel(active, since 2s), standbys: smithi164.bkfwap 2024-09-10T03:30:29.846 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:29 smithi130 ceph-mon[23614]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:30:29.846 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:29 smithi130 ceph-mon[23614]: pgmap v3: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-10T03:30:29.846 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:29 smithi130 ceph-mon[23614]: pgmap v4: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-10T03:30:29.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:29 smithi164 ceph-mon[25711]: mgrmap e31: smithi130.axcmel(active, since 2s), standbys: smithi164.bkfwap 2024-09-10T03:30:29.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:29 smithi164 ceph-mon[25711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:30:29.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:29 smithi164 ceph-mon[25711]: pgmap v3: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-10T03:30:29.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:29 smithi164 ceph-mon[25711]: pgmap v4: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-10T03:30:30.046 INFO:teuthology.orchestra.run.smithi130.stdout: 5 94219 12.66 MB/sec execute 99 sec latency 42.963 ms 2024-09-10T03:30:31.046 INFO:teuthology.orchestra.run.smithi130.stdout: 5 94568 12.64 MB/sec execute 100 sec latency 50.234 ms 2024-09-10T03:30:31.382 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:31 smithi130 ceph-mon[23614]: [10/Sep/2024:03:30:29] ENGINE Bus STARTING 2024-09-10T03:30:31.382 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:31 smithi130 ceph-mon[23614]: [10/Sep/2024:03:30:29] ENGINE Serving on https://172.21.15.130:7150 2024-09-10T03:30:31.382 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:31 smithi130 ceph-mon[23614]: [10/Sep/2024:03:30:29] ENGINE Error in HTTPServer.serve 2024-09-10T03:30:31.382 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Traceback (most recent call last): 2024-09-10T03:30:31.382 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-10T03:30:31.382 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: self._connections.run(self.expiration_interval) 2024-09-10T03:30:31.382 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-10T03:30:31.382 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: self._run(expiration_interval) 2024-09-10T03:30:31.382 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-10T03:30:31.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-10T03:30:31.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-10T03:30:31.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-10T03:30:31.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-10T03:30:31.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: s = self.context.wrap_socket( 2024-09-10T03:30:31.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-10T03:30:31.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: return self.sslsocket_class._create( 2024-09-10T03:30:31.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-10T03:30:31.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: self.do_handshake() 2024-09-10T03:30:31.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-10T03:30:31.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: self._sslobj.do_handshake() 2024-09-10T03:30:31.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-10T03:30:31.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:31 smithi130 ceph-mon[23614]: [10/Sep/2024:03:30:29] ENGINE Serving on http://172.21.15.130:8765 2024-09-10T03:30:31.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:31 smithi130 ceph-mon[23614]: [10/Sep/2024:03:30:29] ENGINE Bus STARTED 2024-09-10T03:30:31.384 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:31 smithi130 ceph-mon[23614]: mgrmap e32: smithi130.axcmel(active, since 4s), standbys: smithi164.bkfwap 2024-09-10T03:30:31.433 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:31 smithi164 ceph-mon[25711]: [10/Sep/2024:03:30:29] ENGINE Bus STARTING 2024-09-10T03:30:31.433 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:31 smithi164 ceph-mon[25711]: [10/Sep/2024:03:30:29] ENGINE Serving on https://172.21.15.130:7150 2024-09-10T03:30:31.433 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:31 smithi164 ceph-mon[25711]: [10/Sep/2024:03:30:29] ENGINE Error in HTTPServer.serve 2024-09-10T03:30:31.433 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Traceback (most recent call last): 2024-09-10T03:30:31.433 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-10T03:30:31.433 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: self._connections.run(self.expiration_interval) 2024-09-10T03:30:31.433 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-10T03:30:31.433 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: self._run(expiration_interval) 2024-09-10T03:30:31.433 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-10T03:30:31.433 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-10T03:30:31.433 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-10T03:30:31.434 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-10T03:30:31.434 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-10T03:30:31.434 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: s = self.context.wrap_socket( 2024-09-10T03:30:31.434 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-10T03:30:31.434 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: return self.sslsocket_class._create( 2024-09-10T03:30:31.434 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-10T03:30:31.434 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: self.do_handshake() 2024-09-10T03:30:31.434 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-10T03:30:31.434 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: self._sslobj.do_handshake() 2024-09-10T03:30:31.434 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-10T03:30:31.434 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:31 smithi164 ceph-mon[25711]: [10/Sep/2024:03:30:29] ENGINE Serving on http://172.21.15.130:8765 2024-09-10T03:30:31.434 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:31 smithi164 ceph-mon[25711]: [10/Sep/2024:03:30:29] ENGINE Bus STARTED 2024-09-10T03:30:31.434 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:31 smithi164 ceph-mon[25711]: mgrmap e32: smithi130.axcmel(active, since 4s), standbys: smithi164.bkfwap 2024-09-10T03:30:32.046 INFO:teuthology.orchestra.run.smithi130.stdout: 5 95009 12.70 MB/sec execute 101 sec latency 39.460 ms 2024-09-10T03:30:32.391 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:32 smithi130 ceph-mon[23614]: pgmap v5: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-10T03:30:32.454 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:32 smithi164 ceph-mon[25711]: pgmap v5: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-10T03:30:33.046 INFO:teuthology.orchestra.run.smithi130.stdout: 5 95446 12.71 MB/sec execute 102 sec latency 54.912 ms 2024-09-10T03:30:33.428 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:33 smithi130 ceph-mon[23614]: mgrmap e33: smithi130.axcmel(active, since 5s), standbys: smithi164.bkfwap 2024-09-10T03:30:33.428 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:33 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:33.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:33 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:33.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:33 smithi164 ceph-mon[25711]: mgrmap e33: smithi130.axcmel(active, since 5s), standbys: smithi164.bkfwap 2024-09-10T03:30:33.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:33 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:33.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:33 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:34.046 INFO:teuthology.orchestra.run.smithi130.stdout: 5 95746 12.70 MB/sec execute 103 sec latency 46.184 ms 2024-09-10T03:30:34.210 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:34 smithi130 ceph-mon[23614]: pgmap v6: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-10T03:30:34.210 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:34 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:34.210 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:34 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:34.210 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:34 smithi130 ceph-mon[23614]: Standby manager daemon smithi164.bkfwap restarted 2024-09-10T03:30:34.210 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:34 smithi130 ceph-mon[23614]: Standby manager daemon smithi164.bkfwap started 2024-09-10T03:30:34.210 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:34 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.164:0/557314653' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/crt"}]: dispatch 2024-09-10T03:30:34.210 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:34 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.164:0/557314653' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-10T03:30:34.210 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:34 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.164:0/557314653' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/key"}]: dispatch 2024-09-10T03:30:34.211 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:34 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.164:0/557314653' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-10T03:30:34.455 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:34 smithi164 ceph-mon[25711]: pgmap v6: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-10T03:30:34.456 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:34 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:34.456 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:34 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:34.456 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:34 smithi164 ceph-mon[25711]: Standby manager daemon smithi164.bkfwap restarted 2024-09-10T03:30:34.456 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:34 smithi164 ceph-mon[25711]: Standby manager daemon smithi164.bkfwap started 2024-09-10T03:30:34.456 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:34 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/557314653' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/crt"}]: dispatch 2024-09-10T03:30:34.456 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:34 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/557314653' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-10T03:30:34.456 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:34 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/557314653' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/key"}]: dispatch 2024-09-10T03:30:34.456 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:34 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/557314653' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-10T03:30:35.046 INFO:teuthology.orchestra.run.smithi130.stdout: 5 95981 12.65 MB/sec execute 104 sec latency 64.649 ms 2024-09-10T03:30:35.283 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:30:35.725 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:35 smithi130 ceph-mon[23614]: mgrmap e34: smithi130.axcmel(active, since 7s), standbys: smithi164.bkfwap 2024-09-10T03:30:35.725 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:35 smithi130 ceph-mon[23614]: from='client.14812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:30:35.725 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:35 smithi130 ceph-mon[23614]: pgmap v7: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 4.4 MiB/s wr, 413 op/s 2024-09-10T03:30:35.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:35 smithi164 ceph-mon[25711]: mgrmap e34: smithi130.axcmel(active, since 7s), standbys: smithi164.bkfwap 2024-09-10T03:30:35.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:35 smithi164 ceph-mon[25711]: from='client.14812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:30:35.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:35 smithi164 ceph-mon[25711]: pgmap v7: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 4.4 MiB/s wr, 413 op/s 2024-09-10T03:30:36.046 INFO:teuthology.orchestra.run.smithi130.stdout: 5 96219 12.55 MB/sec execute 105 sec latency 75.998 ms 2024-09-10T03:30:36.397 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:30:36.398 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (81s) 17s ago 10m 20.0M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:30:36.398 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (10m) 17s ago 10m 6400k - 17.2.0 e1d6a67b021e 138ed3222ba5 2024-09-10T03:30:36.398 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (9m) 3s ago 9m 6405k - 17.2.0 e1d6a67b021e 3623793da55d 2024-09-10T03:30:36.398 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (72s) 17s ago 10m 48.0M - dad864ee21e9 72ba01ac1871 2024-09-10T03:30:36.398 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (4m) 17s ago 4m 35.8M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:30:36.398 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (4m) 3s ago 4m 15.0M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:30:36.398 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (27s) 17s ago 11m 87.9M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:30:36.398 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283 running (2m) 3s ago 9m 145M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e9559882d6ad 2024-09-10T03:30:36.398 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (11m) 17s ago 11m 52.6M 2048M 17.2.0 e1d6a67b021e f2b698da8d6e 2024-09-10T03:30:36.398 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (9m) 3s ago 9m 46.3M 2048M 17.2.0 e1d6a67b021e 7a928b0b045c 2024-09-10T03:30:36.398 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (3m) 17s ago 3m 254M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:30:36.398 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (3m) 3s ago 3m 83.6M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:30:36.399 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (65s) 17s ago 10m 8140k - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:30:36.399 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (37s) 3s ago 9m 9.82M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:30:36.399 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (8m) 17s ago 8m 369M 1936M 17.2.0 e1d6a67b021e 1d66043abecd 2024-09-10T03:30:36.399 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (8m) 17s ago 8m 267M 1936M 17.2.0 e1d6a67b021e ca1ec04aecfa 2024-09-10T03:30:36.399 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (7m) 17s ago 7m 221M 1936M 17.2.0 e1d6a67b021e 5883f5d5e969 2024-09-10T03:30:36.399 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (7m) 17s ago 7m 319M 1936M 17.2.0 e1d6a67b021e b32084e1a404 2024-09-10T03:30:36.399 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (7m) 3s ago 7m 359M 2704M 17.2.0 e1d6a67b021e 3b5c9f00f64d 2024-09-10T03:30:36.399 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (6m) 3s ago 6m 378M 2704M 17.2.0 e1d6a67b021e f16357ac331e 2024-09-10T03:30:36.399 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (6m) 3s ago 6m 312M 2704M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:30:36.399 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (5m) 3s ago 5m 455M 2704M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:30:36.399 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (43s) 17s ago 9m 37.8M - 2.51.0 1d3b7f56885b fe56454650dd 2024-09-10T03:30:37.047 INFO:teuthology.orchestra.run.smithi130.stdout: 5 96541 12.50 MB/sec execute 106 sec latency 38.869 ms 2024-09-10T03:30:37.064 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:30:37.064 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:30:37.064 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:30:37.064 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:30:37.064 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:30:37.064 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:30:37.065 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:30:37.065 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:30:37.065 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-10T03:30:37.065 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:30:37.065 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:30:37.065 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:30:37.065 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:30:37.065 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:30:37.065 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-10T03:30:37.065 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:30:37.065 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:30:37.065 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:30:37.255 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:37 smithi130 ceph-mon[23614]: from='client.14816 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:30:37.255 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:37 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3065552876' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-10T03:30:37.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:37 smithi164 ceph-mon[25711]: from='client.14816 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:30:37.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:37 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/3065552876' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-10T03:30:37.547 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:30:37.547 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:30:37.547 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:30:37.547 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:30:37.547 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-09-10T03:30:37.547 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-09-10T03:30:37.547 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-09-10T03:30:37.547 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "2/23 daemons upgraded", 2024-09-10T03:30:37.547 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "", 2024-09-10T03:30:37.548 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:30:37.548 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:30:38.047 INFO:teuthology.orchestra.run.smithi130.stdout: 5 96828 12.46 MB/sec execute 107 sec latency 43.360 ms 2024-09-10T03:30:38.238 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-09-10T03:30:38.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:38 smithi130 ceph-mon[23614]: from='client.24551 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:30:38.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:38 smithi130 ceph-mon[23614]: pgmap v8: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 3.4 MiB/s wr, 316 op/s 2024-09-10T03:30:38.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:38 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:38.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:38 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:38.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:38 smithi164 ceph-mon[25711]: from='client.24551 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:30:38.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:38 smithi164 ceph-mon[25711]: pgmap v8: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 3.4 MiB/s wr, 316 op/s 2024-09-10T03:30:38.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:38 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:38.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:38 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:39.047 INFO:teuthology.orchestra.run.smithi130.stdout: 5 97174 12.42 MB/sec execute 108 sec latency 63.684 ms 2024-09-10T03:30:39.282 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:39 smithi130 ceph-mon[23614]: from='client.14828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:30:39.283 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:39 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/2788477900' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-10T03:30:39.283 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:39 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:39.283 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:39 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:39.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:39 smithi164 ceph-mon[25711]: from='client.14828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:30:39.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:39 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/2788477900' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-10T03:30:39.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:39 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:39.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:39 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:40.047 INFO:teuthology.orchestra.run.smithi130.stdout: 5 97638 12.36 MB/sec execute 109 sec latency 55.337 ms 2024-09-10T03:30:40.301 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:40 smithi130 ceph-mon[23614]: pgmap v9: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.9 MiB/s wr, 468 op/s 2024-09-10T03:30:40.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:40 smithi164 ceph-mon[25711]: pgmap v9: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.9 MiB/s wr, 468 op/s 2024-09-10T03:30:41.047 INFO:teuthology.orchestra.run.smithi130.stdout: 5 97970 12.31 MB/sec execute 110 sec latency 46.669 ms 2024-09-10T03:30:41.700 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:41 smithi130 ceph-mon[23614]: pgmap v10: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.3 MiB/s wr, 413 op/s 2024-09-10T03:30:41.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:41 smithi164 ceph-mon[25711]: pgmap v10: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.3 MiB/s wr, 413 op/s 2024-09-10T03:30:42.047 INFO:teuthology.orchestra.run.smithi130.stdout: 5 98339 12.35 MB/sec execute 111 sec latency 82.584 ms 2024-09-10T03:30:43.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 98790 12.37 MB/sec execute 112 sec latency 43.721 ms 2024-09-10T03:30:43.237 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:43 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:30:43.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:43 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:30:44.047 INFO:teuthology.orchestra.run.smithi130.stdout: 5 99222 12.39 MB/sec execute 113 sec latency 47.654 ms 2024-09-10T03:30:44.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:44 smithi130 ceph-mon[23614]: pgmap v11: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.3 MiB/s wr, 413 op/s 2024-09-10T03:30:44.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:44 smithi164 ceph-mon[25711]: pgmap v11: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.3 MiB/s wr, 413 op/s 2024-09-10T03:30:45.047 INFO:teuthology.orchestra.run.smithi130.stdout: 5 99564 12.38 MB/sec execute 114 sec latency 35.186 ms 2024-09-10T03:30:46.047 INFO:teuthology.orchestra.run.smithi130.stdout: 5 99894 12.31 MB/sec execute 115 sec latency 32.161 ms 2024-09-10T03:30:46.279 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:46 smithi130 ceph-mon[23614]: pgmap v12: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.7 MiB/s wr, 643 op/s 2024-09-10T03:30:46.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:46 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:46.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:46 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:46.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:46 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:30:46.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:46 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:30:46.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:46 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:46.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:46 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:46.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:46 smithi164 ceph-mon[25711]: pgmap v12: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.7 MiB/s wr, 643 op/s 2024-09-10T03:30:46.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:46 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:46.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:46 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:46.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:46 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:30:46.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:46 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:30:46.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:46 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:46.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:46 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:47.047 INFO:teuthology.orchestra.run.smithi130.stdout: 5 100205 12.27 MB/sec execute 116 sec latency 31.194 ms 2024-09-10T03:30:47.296 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:47 smithi130 ceph-mon[23614]: Updating smithi130:/etc/ceph/ceph.conf 2024-09-10T03:30:47.296 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:47 smithi130 ceph-mon[23614]: Updating smithi164:/etc/ceph/ceph.conf 2024-09-10T03:30:47.296 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:47 smithi130 ceph-mon[23614]: Updating smithi164:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.conf 2024-09-10T03:30:47.296 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:47 smithi130 ceph-mon[23614]: Updating smithi130:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.conf 2024-09-10T03:30:47.296 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:47 smithi130 ceph-mon[23614]: Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2024-09-10T03:30:47.296 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:47 smithi130 ceph-mon[23614]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-09-10T03:30:47.296 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:47 smithi130 ceph-mon[23614]: Updating smithi164:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.client.admin.keyring 2024-09-10T03:30:47.296 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:47 smithi130 ceph-mon[23614]: Updating smithi130:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.client.admin.keyring 2024-09-10T03:30:47.296 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:47 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:47.296 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:47 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:47.297 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:47 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:47.297 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:47 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:47.297 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:47 smithi130 ceph-mon[23614]: Reconfiguring prometheus.smithi130 (dependencies changed)... 2024-09-10T03:30:47.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:47 smithi164 ceph-mon[25711]: Updating smithi130:/etc/ceph/ceph.conf 2024-09-10T03:30:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:47 smithi164 ceph-mon[25711]: Updating smithi164:/etc/ceph/ceph.conf 2024-09-10T03:30:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:47 smithi164 ceph-mon[25711]: Updating smithi164:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.conf 2024-09-10T03:30:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:47 smithi164 ceph-mon[25711]: Updating smithi130:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.conf 2024-09-10T03:30:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:47 smithi164 ceph-mon[25711]: Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2024-09-10T03:30:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:47 smithi164 ceph-mon[25711]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-09-10T03:30:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:47 smithi164 ceph-mon[25711]: Updating smithi164:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.client.admin.keyring 2024-09-10T03:30:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:47 smithi164 ceph-mon[25711]: Updating smithi130:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.client.admin.keyring 2024-09-10T03:30:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:47 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:47 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:47 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:47 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:47 smithi164 ceph-mon[25711]: Reconfiguring prometheus.smithi130 (dependencies changed)... 2024-09-10T03:30:48.047 INFO:teuthology.orchestra.run.smithi130.stdout: 5 100469 12.22 MB/sec execute 117 sec latency 67.705 ms 2024-09-10T03:30:48.310 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:48 smithi130 ceph-mon[23614]: Reconfiguring daemon prometheus.smithi130 on smithi130 2024-09-10T03:30:48.311 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:48 smithi130 ceph-mon[23614]: pgmap v13: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.2 MiB/s wr, 416 op/s 2024-09-10T03:30:48.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:48 smithi164 ceph-mon[25711]: Reconfiguring daemon prometheus.smithi130 on smithi130 2024-09-10T03:30:48.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:48 smithi164 ceph-mon[25711]: pgmap v13: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.2 MiB/s wr, 416 op/s 2024-09-10T03:30:49.047 INFO:teuthology.orchestra.run.smithi130.stdout: 5 100931 12.21 MB/sec execute 118 sec latency 37.920 ms 2024-09-10T03:30:49.588 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:49 smithi130 ceph-mon[23614]: pgmap v14: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.3 MiB/s wr, 638 op/s 2024-09-10T03:30:49.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:49 smithi164 ceph-mon[25711]: pgmap v14: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.3 MiB/s wr, 638 op/s 2024-09-10T03:30:50.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 101319 12.14 MB/sec execute 119 sec latency 50.177 ms 2024-09-10T03:30:51.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 101612 12.12 MB/sec execute 120 sec latency 48.329 ms 2024-09-10T03:30:51.608 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:51 smithi130 ceph-mon[23614]: pgmap v15: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.4 MiB/s wr, 451 op/s 2024-09-10T03:30:51.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:51 smithi164 ceph-mon[25711]: pgmap v15: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.4 MiB/s wr, 451 op/s 2024-09-10T03:30:52.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 101994 12.15 MB/sec execute 121 sec latency 35.599 ms 2024-09-10T03:30:53.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 102448 12.16 MB/sec execute 122 sec latency 53.838 ms 2024-09-10T03:30:53.649 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:53 smithi130 ceph-mon[23614]: pgmap v16: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.4 MiB/s wr, 451 op/s 2024-09-10T03:30:53.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:53 smithi164 ceph-mon[25711]: pgmap v16: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.4 MiB/s wr, 451 op/s 2024-09-10T03:30:54.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 102831 12.19 MB/sec execute 123 sec latency 111.685 ms 2024-09-10T03:30:55.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 103054 12.12 MB/sec execute 124 sec latency 75.175 ms 2024-09-10T03:30:56.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 103340 12.07 MB/sec execute 125 sec latency 113.961 ms 2024-09-10T03:30:56.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:56 smithi130 ceph-mon[23614]: pgmap v17: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 672 op/s 2024-09-10T03:30:56.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:56 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:56.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:56 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:56.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:56 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-10T03:30:56.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:56 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:30:56.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:56 smithi164 ceph-mon[25711]: pgmap v17: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 672 op/s 2024-09-10T03:30:56.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:56 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:56.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:56 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:30:56.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:56 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-10T03:30:56.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:56 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:30:57.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 104192 12.16 MB/sec execute 126 sec latency 35.493 ms 2024-09-10T03:30:57.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:57 smithi130 ceph-mon[23614]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-10T03:30:57.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:57 smithi164 ceph-mon[25711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-10T03:30:58.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 104873 12.14 MB/sec execute 127 sec latency 32.409 ms 2024-09-10T03:30:58.437 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:58 smithi130 ceph-mon[23614]: Upgrade: Updating mgr.smithi164.bkfwap 2024-09-10T03:30:58.437 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:58 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.bkfwap", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-10T03:30:58.437 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:58 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.bkfwap", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-10T03:30:58.437 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:58 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-10T03:30:58.437 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:58 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:30:58.437 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:58 smithi130 ceph-mon[23614]: Deploying daemon mgr.smithi164.bkfwap on smithi164 2024-09-10T03:30:58.437 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:58 smithi130 ceph-mon[23614]: pgmap v18: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 441 op/s 2024-09-10T03:30:58.437 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:58 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:30:58.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:58 smithi164 ceph-mon[25711]: Upgrade: Updating mgr.smithi164.bkfwap 2024-09-10T03:30:58.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:58 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.bkfwap", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-10T03:30:58.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:58 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi164.bkfwap", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-10T03:30:58.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:58 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-10T03:30:58.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:58 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:30:58.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:58 smithi164 ceph-mon[25711]: Deploying daemon mgr.smithi164.bkfwap on smithi164 2024-09-10T03:30:58.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:58 smithi164 ceph-mon[25711]: pgmap v18: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 441 op/s 2024-09-10T03:30:58.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:58 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:30:59.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 105377 12.16 MB/sec execute 128 sec latency 44.679 ms 2024-09-10T03:30:59.453 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:30:59 smithi130 ceph-mon[23614]: pgmap v19: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.4 MiB/s wr, 741 op/s 2024-09-10T03:30:59.698 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:30:59 smithi164 ceph-mon[25711]: pgmap v19: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.4 MiB/s wr, 741 op/s 2024-09-10T03:31:00.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 105896 12.23 MB/sec execute 129 sec latency 54.508 ms 2024-09-10T03:31:01.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 106255 12.23 MB/sec execute 130 sec latency 48.704 ms 2024-09-10T03:31:02.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 106455 12.18 MB/sec execute 131 sec latency 55.560 ms 2024-09-10T03:31:02.550 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:02 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:02.550 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:02 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:02.551 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:02 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:31:02.551 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:02 smithi130 ceph-mon[23614]: pgmap v20: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.3 MiB/s wr, 520 op/s 2024-09-10T03:31:02.551 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:02 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:02.551 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:02 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:02.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:02 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:02.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:02 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:02.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:02 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:31:02.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:02 smithi164 ceph-mon[25711]: pgmap v20: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.3 MiB/s wr, 520 op/s 2024-09-10T03:31:02.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:02 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:02.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:02 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:03.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 106670 12.14 MB/sec execute 132 sec latency 134.846 ms 2024-09-10T03:31:03.586 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:03 smithi164 ceph-mon[25711]: pgmap v21: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.3 MiB/s wr, 520 op/s 2024-09-10T03:31:03.625 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:03 smithi130 ceph-mon[23614]: pgmap v21: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.3 MiB/s wr, 520 op/s 2024-09-10T03:31:04.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 106876 12.06 MB/sec execute 133 sec latency 101.454 ms 2024-09-10T03:31:05.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 107088 12.00 MB/sec execute 134 sec latency 77.948 ms 2024-09-10T03:31:06.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 107338 11.98 MB/sec execute 135 sec latency 50.666 ms 2024-09-10T03:31:06.212 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:06 smithi130 ceph-mon[23614]: pgmap v22: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 720 op/s 2024-09-10T03:31:06.212 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:06 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:06.212 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:06 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:06.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:06 smithi164 ceph-mon[25711]: pgmap v22: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 720 op/s 2024-09-10T03:31:06.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:06 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:06.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:06 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:07.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 107603 11.94 MB/sec execute 136 sec latency 74.204 ms 2024-09-10T03:31:07.509 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:07 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:07.509 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:07 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:07.509 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:07 smithi130 ceph-mon[23614]: pgmap v23: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.2 MiB/s wr, 500 op/s 2024-09-10T03:31:07.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:07 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:07.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:07 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:07.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:07 smithi164 ceph-mon[25711]: pgmap v23: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.2 MiB/s wr, 500 op/s 2024-09-10T03:31:08.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 108015 11.92 MB/sec execute 137 sec latency 48.446 ms 2024-09-10T03:31:08.517 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:08 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.164:0/1001689529' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/crt"}]: dispatch 2024-09-10T03:31:08.517 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:08 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.164:0/1001689529' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-10T03:31:08.517 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:08 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.164:0/1001689529' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/key"}]: dispatch 2024-09-10T03:31:08.517 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:08 smithi130 ceph-mon[23614]: from='mgr.? 172.21.15.164:0/1001689529' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-10T03:31:08.517 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:08 smithi130 ceph-mon[23614]: Standby manager daemon smithi164.bkfwap restarted 2024-09-10T03:31:08.517 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:08 smithi130 ceph-mon[23614]: Standby manager daemon smithi164.bkfwap started 2024-09-10T03:31:08.810 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:08 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/1001689529' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/crt"}]: dispatch 2024-09-10T03:31:08.810 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:08 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/1001689529' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-10T03:31:08.810 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:08 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/1001689529' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/key"}]: dispatch 2024-09-10T03:31:08.810 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:08 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/1001689529' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-10T03:31:08.810 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:08 smithi164 ceph-mon[25711]: Standby manager daemon smithi164.bkfwap restarted 2024-09-10T03:31:08.811 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:08 smithi164 ceph-mon[25711]: Standby manager daemon smithi164.bkfwap started 2024-09-10T03:31:08.864 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:31:09.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 108402 11.86 MB/sec execute 138 sec latency 63.088 ms 2024-09-10T03:31:09.777 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:09 smithi130 ceph-mon[23614]: mgrmap e35: smithi130.axcmel(active, since 41s), standbys: smithi164.bkfwap 2024-09-10T03:31:09.777 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:09 smithi130 ceph-mon[23614]: from='client.24567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:31:09.777 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:09 smithi130 ceph-mon[23614]: pgmap v24: 97 pgs: 97 active+clean; 321 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 685 op/s 2024-09-10T03:31:09.822 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:09 smithi164 ceph-mon[25711]: mgrmap e35: smithi130.axcmel(active, since 41s), standbys: smithi164.bkfwap 2024-09-10T03:31:09.822 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:09 smithi164 ceph-mon[25711]: from='client.24567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:31:09.822 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:09 smithi164 ceph-mon[25711]: pgmap v24: 97 pgs: 97 active+clean; 321 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 685 op/s 2024-09-10T03:31:09.896 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:31:09.896 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (115s) 32s ago 10m 20.0M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:31:09.897 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (10m) 32s ago 10m 6400k - 17.2.0 e1d6a67b021e 138ed3222ba5 2024-09-10T03:31:09.897 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (10m) 4s ago 10m 6405k - 17.2.0 e1d6a67b021e 3623793da55d 2024-09-10T03:31:09.897 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (106s) 32s ago 10m 48.0M - dad864ee21e9 72ba01ac1871 2024-09-10T03:31:09.897 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (4m) 32s ago 4m 36.2M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:31:09.897 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (4m) 4s ago 4m 15.2M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:31:09.897 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (61s) 32s ago 12m 490M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:31:09.897 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283,8765 running (9s) 4s ago 10m 86.6M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e4034e8e2a6d 2024-09-10T03:31:09.897 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (12m) 32s ago 12m 57.5M 2048M 17.2.0 e1d6a67b021e f2b698da8d6e 2024-09-10T03:31:09.897 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (10m) 4s ago 10m 45.6M 2048M 17.2.0 e1d6a67b021e 7a928b0b045c 2024-09-10T03:31:09.897 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (4m) 32s ago 4m 254M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:31:09.897 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (4m) 4s ago 4m 85.7M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:31:09.897 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (98s) 32s ago 10m 8056k - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:31:09.898 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (71s) 4s ago 10m 10.2M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:31:09.898 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (9m) 32s ago 9m 387M 1936M 17.2.0 e1d6a67b021e 1d66043abecd 2024-09-10T03:31:09.898 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (8m) 32s ago 8m 287M 1936M 17.2.0 e1d6a67b021e ca1ec04aecfa 2024-09-10T03:31:09.898 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (8m) 32s ago 8m 247M 1936M 17.2.0 e1d6a67b021e 5883f5d5e969 2024-09-10T03:31:09.898 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (7m) 32s ago 7m 359M 1936M 17.2.0 e1d6a67b021e b32084e1a404 2024-09-10T03:31:09.898 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (7m) 4s ago 7m 388M 2704M 17.2.0 e1d6a67b021e 3b5c9f00f64d 2024-09-10T03:31:09.898 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (7m) 4s ago 7m 413M 2704M 17.2.0 e1d6a67b021e f16357ac331e 2024-09-10T03:31:09.898 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (6m) 4s ago 6m 350M 2704M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:31:09.898 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (6m) 4s ago 6m 502M 2704M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:31:09.898 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 starting - - - - 2024-09-10T03:31:10.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 108693 11.84 MB/sec execute 139 sec latency 72.471 ms 2024-09-10T03:31:10.519 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:31:10.520 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:31:10.520 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:31:10.520 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:31:10.520 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:31:10.520 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:31:10.520 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:31:10.520 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:31:10.520 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-10T03:31:10.520 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:31:10.520 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:31:10.520 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:31:10.521 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:31:10.521 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:31:10.521 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-10T03:31:10.521 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:31:10.521 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:31:10.521 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:31:11.048 INFO:teuthology.orchestra.run.smithi130.stdout: 5 109013 11.84 MB/sec execute 140 sec latency 89.757 ms 2024-09-10T03:31:11.052 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:31:11.052 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:31:11.052 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:31:11.052 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:31:11.052 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-09-10T03:31:11.053 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-09-10T03:31:11.053 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-09-10T03:31:11.053 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "2/23 daemons upgraded", 2024-09-10T03:31:11.053 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading mon daemons", 2024-09-10T03:31:11.053 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:31:11.053 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:31:11.333 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='client.14842 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:31:11.333 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='client.14846 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:31:11.333 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:11.333 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:11.333 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:11.333 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/1944755471' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-10T03:31:11.333 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:11.334 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:31:11.334 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:31:11.334 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:11.334 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:11.334 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:31:11.334 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "versions"}]: dispatch 2024-09-10T03:31:11.334 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:11.334 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi130.axcmel"}]: dispatch 2024-09-10T03:31:11.334 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi130.axcmel"}]: dispatch 2024-09-10T03:31:11.334 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi130.axcmel"}]': finished 2024-09-10T03:31:11.334 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi164.bkfwap"}]: dispatch 2024-09-10T03:31:11.334 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi164.bkfwap"}]: dispatch 2024-09-10T03:31:11.334 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi164.bkfwap"}]': finished 2024-09-10T03:31:11.335 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:11 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-10T03:31:11.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='client.14842 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:31:11.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='client.14846 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:31:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/1944755471' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-10T03:31:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:31:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-10T03:31:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-10T03:31:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "versions"}]: dispatch 2024-09-10T03:31:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi130.axcmel"}]: dispatch 2024-09-10T03:31:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi130.axcmel"}]: dispatch 2024-09-10T03:31:11.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi130.axcmel"}]': finished 2024-09-10T03:31:11.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi164.bkfwap"}]: dispatch 2024-09-10T03:31:11.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi164.bkfwap"}]: dispatch 2024-09-10T03:31:11.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi164.bkfwap"}]': finished 2024-09-10T03:31:11.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:11 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-10T03:31:11.702 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-09-10T03:31:12.049 INFO:teuthology.orchestra.run.smithi130.stdout: 5 109439 11.87 MB/sec execute 141 sec latency 75.718 ms 2024-09-10T03:31:12.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:12 smithi130 ceph-mon[23614]: Upgrade: Setting container_image for all mgr 2024-09-10T03:31:12.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:12 smithi130 ceph-mon[23614]: from='client.24581 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:31:12.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:12 smithi130 ceph-mon[23614]: pgmap v25: 97 pgs: 97 active+clean; 321 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.7 MiB/s wr, 385 op/s 2024-09-10T03:31:12.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:12 smithi130 ceph-mon[23614]: from='client.? 172.21.15.130:0/3532047867' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-10T03:31:12.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:12 smithi164 ceph-mon[25711]: Upgrade: Setting container_image for all mgr 2024-09-10T03:31:12.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:12 smithi164 ceph-mon[25711]: from='client.24581 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:31:12.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:12 smithi164 ceph-mon[25711]: pgmap v25: 97 pgs: 97 active+clean; 321 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.7 MiB/s wr, 385 op/s 2024-09-10T03:31:12.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:12 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/3532047867' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-10T03:31:13.049 INFO:teuthology.orchestra.run.smithi130.stdout: 5 109842 11.87 MB/sec execute 142 sec latency 64.562 ms 2024-09-10T03:31:13.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:13 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:31:13.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:13 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-10T03:31:14.049 INFO:teuthology.orchestra.run.smithi130.stdout: 5 110025 11.83 MB/sec execute 143 sec latency 68.306 ms 2024-09-10T03:31:14.388 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:14 smithi130 ceph-mon[23614]: Upgrade: Updating mon.smithi130 2024-09-10T03:31:14.388 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:14 smithi130 ceph-mon[23614]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:14.389 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:14 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-10T03:31:14.389 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:14 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-10T03:31:14.389 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:14 smithi130 ceph-mon[23614]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:31:14.389 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:14 smithi130 ceph-mon[23614]: Deploying daemon mon.smithi130 on smithi130 2024-09-10T03:31:14.389 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:14 smithi130 ceph-mon[23614]: pgmap v26: 97 pgs: 97 active+clean; 321 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.7 MiB/s wr, 385 op/s 2024-09-10T03:31:14.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:14 smithi164 ceph-mon[25711]: Upgrade: Updating mon.smithi130 2024-09-10T03:31:14.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:14 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:14.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:14 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-10T03:31:14.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:14 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-10T03:31:14.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:14 smithi164 ceph-mon[25711]: from='mgr.14782 172.21.15.130:0/2150905282' entity='mgr.smithi130.axcmel' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-10T03:31:14.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:14 smithi164 ceph-mon[25711]: Deploying daemon mon.smithi130 on smithi130 2024-09-10T03:31:14.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:14 smithi164 ceph-mon[25711]: pgmap v26: 97 pgs: 97 active+clean; 321 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.7 MiB/s wr, 385 op/s 2024-09-10T03:31:15.049 INFO:teuthology.orchestra.run.smithi130.stdout: 5 110259 11.79 MB/sec execute 144 sec latency 65.628 ms 2024-09-10T03:31:16.049 INFO:teuthology.orchestra.run.smithi130.stdout: 5 110554 11.74 MB/sec execute 145 sec latency 67.607 ms 2024-09-10T03:31:16.217 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:16 smithi130 systemd[1]: Stopping Ceph mon.smithi130 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:31:16.217 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:16 smithi130 ceph-mon[23614]: pgmap v27: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.2 MiB/s wr, 585 op/s 2024-09-10T03:31:16.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:16 smithi164 ceph-mon[25711]: pgmap v27: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.2 MiB/s wr, 585 op/s 2024-09-10T03:31:16.723 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:16 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130[23610]: 2024-09-10T03:31:16.624+0000 7f8eeffb8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi130 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:31:16.723 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:16 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130[23610]: 2024-09-10T03:31:16.624+0000 7f8eeffb8700 -1 mon.smithi130@0(leader) e2 *** Got Signal Terminated *** 2024-09-10T03:31:16.723 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:16 smithi130 podman[63438]: 2024-09-10 03:31:16.701056978 +0000 UTC m=+0.351431677 container died f2b698da8d6e9e2b1300afb4ba700a2d2899100eac9ae16ddade4563f4b7527c (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.19.8, architecture=x86_64, ceph=True, name=centos-stream, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, distribution-scope=public, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-10T03:31:17.049 INFO:teuthology.orchestra.run.smithi130.stdout: 5 110996 11.75 MB/sec execute 146 sec latency 68.133 ms 2024-09-10T03:31:18.049 INFO:teuthology.orchestra.run.smithi130.stdout: 5 111472 11.75 MB/sec execute 147 sec latency 42.092 ms 2024-09-10T03:31:18.060 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:18 smithi130 podman[63438]: 2024-09-10 03:31:18.050600836 +0000 UTC m=+1.700975526 container cleanup f2b698da8d6e9e2b1300afb4ba700a2d2899100eac9ae16ddade4563f4b7527c (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, distribution-scope=public, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vendor=Red Hat, Inc., vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, release=754) 2024-09-10T03:31:18.060 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:18 smithi130 bash[63438]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130 2024-09-10T03:31:18.566 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:18 smithi130 podman[63451]: 2024-09-10 03:31:18.49533745 +0000 UTC m=+1.785230927 container remove f2b698da8d6e9e2b1300afb4ba700a2d2899100eac9ae16ddade4563f4b7527c (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130, release=754, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.buildah.version=1.19.8, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-10T03:31:19.059 INFO:teuthology.orchestra.run.smithi130.stdout: 5 112054 11.71 MB/sec execute 148 sec latency 64.715 ms 2024-09-10T03:31:19.069 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:18 smithi130 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@mon.smithi130.service: Deactivated successfully. 2024-09-10T03:31:19.069 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:18 smithi130 systemd[1]: Stopped Ceph mon.smithi130 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:31:19.069 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:18 smithi130 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@mon.smithi130.service: Consumed 15.289s CPU time. 2024-09-10T03:31:19.323 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:19 smithi130 systemd[1]: Starting Ceph mon.smithi130 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:31:19.573 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:19 smithi130 podman[63572]: 2024-09-10 03:31:19.469050853 +0000 UTC m=+0.028772903 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:31:19.825 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:19 smithi130 podman[63572]: 2024-09-10 03:31:19.676974955 +0000 UTC m=+0.236696985 container create 22e5cf56527a49da6464cb3ff752d169e9c5add018f7978f02b85e633b6f35ce (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:31:20.059 INFO:teuthology.orchestra.run.smithi130.stdout: 5 112561 11.77 MB/sec execute 149 sec latency 42.045 ms 2024-09-10T03:31:20.083 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 podman[63572]: 2024-09-10 03:31:20.077411724 +0000 UTC m=+0.637133772 container init 22e5cf56527a49da6464cb3ff752d169e9c5add018f7978f02b85e633b6f35ce (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-10T03:31:20.083 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 podman[63572]: 2024-09-10 03:31:20.082974346 +0000 UTC m=+0.642696376 container start 22e5cf56527a49da6464cb3ff752d169e9c5add018f7978f02b85e633b6f35ce (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-10T03:31:20.335 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: set uid:gid to 167:167 (ceph:ceph) 2024-09-10T03:31:20.335 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev), process ceph-mon, pid 2 2024-09-10T03:31:20.336 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: pidfile_write: ignore empty --pid-file 2024-09-10T03:31:20.336 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: load: jerasure load: lrc 2024-09-10T03:31:20.336 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: RocksDB version: 7.9.2 2024-09-10T03:31:20.336 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Git sha 0 2024-09-10T03:31:20.336 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Compile date 2024-09-10 00:49:01 2024-09-10T03:31:20.336 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: DB SUMMARY 2024-09-10T03:31:20.336 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: DB Session ID: 9TWG90XKPOS2B44Z70TQ 2024-09-10T03:31:20.336 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: CURRENT file: CURRENT 2024-09-10T03:31:20.336 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: IDENTITY file: IDENTITY 2024-09-10T03:31:20.336 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: MANIFEST file: MANIFEST-000015 size: 978 Bytes 2024-09-10T03:31:20.336 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi130/store.db dir, Total Num: 1, files: 000030.sst 2024-09-10T03:31:20.336 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi130/store.db: 000028.log size: 2868305 ; 2024-09-10T03:31:20.336 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.error_if_exists: 0 2024-09-10T03:31:20.336 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.create_if_missing: 0 2024-09-10T03:31:20.337 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.paranoid_checks: 1 2024-09-10T03:31:20.337 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-10T03:31:20.337 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-10T03:31:20.337 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-10T03:31:20.337 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.env: 0x5570fd2bd120 2024-09-10T03:31:20.337 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.fs: PosixFileSystem 2024-09-10T03:31:20.337 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.info_log: 0x5570ff6d58e0 2024-09-10T03:31:20.338 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_file_opening_threads: 16 2024-09-10T03:31:20.338 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.statistics: (nil) 2024-09-10T03:31:20.338 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.use_fsync: 0 2024-09-10T03:31:20.338 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_log_file_size: 0 2024-09-10T03:31:20.338 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-10T03:31:20.338 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-10T03:31:20.338 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.keep_log_file_num: 1000 2024-09-10T03:31:20.338 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.recycle_log_file_num: 0 2024-09-10T03:31:20.338 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.allow_fallocate: 1 2024-09-10T03:31:20.339 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.allow_mmap_reads: 0 2024-09-10T03:31:20.339 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.allow_mmap_writes: 0 2024-09-10T03:31:20.339 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.use_direct_reads: 0 2024-09-10T03:31:20.339 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-10T03:31:20.339 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.create_missing_column_families: 0 2024-09-10T03:31:20.339 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.db_log_dir: 2024-09-10T03:31:20.339 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.wal_dir: 2024-09-10T03:31:20.339 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-10T03:31:20.340 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-10T03:31:20.340 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-10T03:31:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-10T03:31:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-10T03:31:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-10T03:31:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.advise_random_on_open: 1 2024-09-10T03:31:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.db_write_buffer_size: 0 2024-09-10T03:31:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.write_buffer_manager: 0x5570ff6d9b80 2024-09-10T03:31:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-10T03:31:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-10T03:31:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-10T03:31:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.rate_limiter: (nil) 2024-09-10T03:31:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-10T03:31:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.wal_recovery_mode: 2 2024-09-10T03:31:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.enable_thread_tracking: 0 2024-09-10T03:31:20.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.enable_pipelined_write: 0 2024-09-10T03:31:20.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.unordered_write: 0 2024-09-10T03:31:20.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-10T03:31:20.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-10T03:31:20.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-10T03:31:20.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-10T03:31:20.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.row_cache: None 2024-09-10T03:31:20.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.wal_filter: None 2024-09-10T03:31:20.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-10T03:31:20.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.allow_ingest_behind: 0 2024-09-10T03:31:20.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.two_write_queues: 0 2024-09-10T03:31:20.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.manual_wal_flush: 0 2024-09-10T03:31:20.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.wal_compression: 0 2024-09-10T03:31:20.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.atomic_flush: 0 2024-09-10T03:31:20.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-10T03:31:20.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-10T03:31:20.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-10T03:31:20.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.log_readahead_size: 0 2024-09-10T03:31:20.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-10T03:31:20.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.best_efforts_recovery: 0 2024-09-10T03:31:20.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-10T03:31:20.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-10T03:31:20.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.allow_data_in_errors: 0 2024-09-10T03:31:20.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.db_host_id: __hostname__ 2024-09-10T03:31:20.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-10T03:31:20.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_background_jobs: 2 2024-09-10T03:31:20.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_background_compactions: -1 2024-09-10T03:31:20.343 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_subcompactions: 1 2024-09-10T03:31:20.344 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-10T03:31:20.344 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-10T03:31:20.344 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-10T03:31:20.344 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_total_wal_size: 0 2024-09-10T03:31:20.344 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-10T03:31:20.344 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-10T03:31:20.344 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-10T03:31:20.344 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-10T03:31:20.344 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_open_files: -1 2024-09-10T03:31:20.344 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.bytes_per_sync: 0 2024-09-10T03:31:20.344 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-10T03:31:20.344 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-10T03:31:20.344 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compaction_readahead_size: 0 2024-09-10T03:31:20.344 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_background_flushes: -1 2024-09-10T03:31:20.345 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Compression algorithms supported: 2024-09-10T03:31:20.345 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: kZSTD supported: 0 2024-09-10T03:31:20.345 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: kXpressCompression supported: 0 2024-09-10T03:31:20.345 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: kBZip2Compression supported: 0 2024-09-10T03:31:20.345 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-10T03:31:20.345 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: kLZ4Compression supported: 1 2024-09-10T03:31:20.345 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: kZlibCompression supported: 1 2024-09-10T03:31:20.345 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: kLZ4HCCompression supported: 1 2024-09-10T03:31:20.345 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: kSnappyCompression supported: 1 2024-09-10T03:31:20.345 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-10T03:31:20.345 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-10T03:31:20.345 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi130/store.db/MANIFEST-000015 2024-09-10T03:31:20.345 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-10T03:31:20.346 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-10T03:31:20.346 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.merge_operator: 2024-09-10T03:31:20.346 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compaction_filter: None 2024-09-10T03:31:20.346 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compaction_filter_factory: None 2024-09-10T03:31:20.346 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.sst_partitioner_factory: None 2024-09-10T03:31:20.346 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-10T03:31:20.346 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-10T03:31:20.346 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5570ff6d5540) 2024-09-10T03:31:20.346 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: cache_index_and_filter_blocks: 1 2024-09-10T03:31:20.346 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-10T03:31:20.346 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-10T03:31:20.346 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: pin_top_level_index_and_filter: 1 2024-09-10T03:31:20.346 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: index_type: 0 2024-09-10T03:31:20.347 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: data_block_index_type: 0 2024-09-10T03:31:20.347 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: index_shortening: 1 2024-09-10T03:31:20.347 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-10T03:31:20.347 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: checksum: 4 2024-09-10T03:31:20.347 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: no_block_cache: 0 2024-09-10T03:31:20.347 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: block_cache: 0x5570ff6c9610 2024-09-10T03:31:20.347 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: block_cache_name: BinnedLRUCache 2024-09-10T03:31:20.347 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: block_cache_options: 2024-09-10T03:31:20.347 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: capacity : 536870912 2024-09-10T03:31:20.347 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: num_shard_bits : 4 2024-09-10T03:31:20.347 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: strict_capacity_limit : 0 2024-09-10T03:31:20.347 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: high_pri_pool_ratio: 0.000 2024-09-10T03:31:20.347 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: block_cache_compressed: (nil) 2024-09-10T03:31:20.347 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: persistent_cache: (nil) 2024-09-10T03:31:20.348 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: block_size: 4096 2024-09-10T03:31:20.348 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: block_size_deviation: 10 2024-09-10T03:31:20.348 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: block_restart_interval: 16 2024-09-10T03:31:20.348 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: index_block_restart_interval: 1 2024-09-10T03:31:20.348 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: metadata_block_size: 4096 2024-09-10T03:31:20.348 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: partition_filters: 0 2024-09-10T03:31:20.348 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: use_delta_encoding: 1 2024-09-10T03:31:20.348 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: filter_policy: bloomfilter 2024-09-10T03:31:20.348 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: whole_key_filtering: 1 2024-09-10T03:31:20.348 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: verify_compression: 0 2024-09-10T03:31:20.348 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: read_amp_bytes_per_bit: 0 2024-09-10T03:31:20.348 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: format_version: 5 2024-09-10T03:31:20.348 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: enable_index_compression: 1 2024-09-10T03:31:20.349 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: block_align: 0 2024-09-10T03:31:20.349 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: max_auto_readahead_size: 262144 2024-09-10T03:31:20.349 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: prepopulate_block_cache: 0 2024-09-10T03:31:20.349 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: initial_auto_readahead_size: 8192 2024-09-10T03:31:20.349 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: num_file_reads_for_auto_readahead: 2 2024-09-10T03:31:20.349 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.write_buffer_size: 33554432 2024-09-10T03:31:20.349 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_write_buffer_number: 2 2024-09-10T03:31:20.349 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compression: NoCompression 2024-09-10T03:31:20.349 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.bottommost_compression: Disabled 2024-09-10T03:31:20.349 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.prefix_extractor: nullptr 2024-09-10T03:31:20.349 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-10T03:31:20.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.num_levels: 7 2024-09-10T03:31:20.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-10T03:31:20.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-10T03:31:20.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-10T03:31:20.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-10T03:31:20.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-10T03:31:20.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-10T03:31:20.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-10T03:31:20.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-10T03:31:20.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-10T03:31:20.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-10T03:31:20.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-10T03:31:20.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-10T03:31:20.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-10T03:31:20.351 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compression_opts.level: 32767 2024-09-10T03:31:20.351 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compression_opts.strategy: 0 2024-09-10T03:31:20.351 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-10T03:31:20.351 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-10T03:31:20.351 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-10T03:31:20.351 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-10T03:31:20.351 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compression_opts.enabled: false 2024-09-10T03:31:20.351 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-10T03:31:20.351 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-10T03:31:20.351 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-10T03:31:20.351 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-10T03:31:20.351 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.target_file_size_base: 67108864 2024-09-10T03:31:20.351 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-10T03:31:20.351 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-10T03:31:20.352 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-10T03:31:20.352 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-10T03:31:20.352 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-10T03:31:20.352 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-10T03:31:20.352 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-10T03:31:20.352 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-10T03:31:20.352 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-10T03:31:20.352 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-10T03:31:20.352 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-10T03:31:20.352 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-10T03:31:20.352 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-10T03:31:20.352 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-10T03:31:20.352 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.arena_block_size: 1048576 2024-09-10T03:31:20.352 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-10T03:31:20.353 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-10T03:31:20.353 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.disable_auto_compactions: 0 2024-09-10T03:31:20.353 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-10T03:31:20.353 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-10T03:31:20.353 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-10T03:31:20.353 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-10T03:31:20.353 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-10T03:31:20.353 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-10T03:31:20.353 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-10T03:31:20.353 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-10T03:31:20.353 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-10T03:31:20.353 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-10T03:31:20.353 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-10T03:31:20.354 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.inplace_update_support: 0 2024-09-10T03:31:20.354 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-10T03:31:20.354 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-10T03:31:20.354 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-10T03:31:20.354 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-10T03:31:20.354 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.bloom_locality: 0 2024-09-10T03:31:20.354 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.max_successive_merges: 0 2024-09-10T03:31:20.354 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-10T03:31:20.354 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.paranoid_file_checks: 0 2024-09-10T03:31:20.354 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.force_consistency_checks: 1 2024-09-10T03:31:20.354 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.report_bg_io_stats: 0 2024-09-10T03:31:20.354 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.ttl: 2592000 2024-09-10T03:31:20.354 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-10T03:31:20.354 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-10T03:31:20.354 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-10T03:31:20.355 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.enable_blob_files: false 2024-09-10T03:31:20.355 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.min_blob_size: 0 2024-09-10T03:31:20.355 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.blob_file_size: 268435456 2024-09-10T03:31:20.355 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-10T03:31:20.355 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-10T03:31:20.355 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-10T03:31:20.355 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-10T03:31:20.355 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-10T03:31:20.355 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.blob_file_starting_level: 0 2024-09-10T03:31:20.355 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-10T03:31:20.355 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-09-10T03:31:20.355 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-10T03:31:20.355 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi130/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11827, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-10T03:31:20.356 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-10T03:31:20.356 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 22d0da89-8c83-45f7-a40a-8a3fa16b036a 2024-09-10T03:31:20.356 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725939080119704, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-10T03:31:20.356 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-10T03:31:20.356 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725939080130455, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2659657, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11828, "largest_seqno": 12844, "table_properties": {"data_size": 2654334, "index_size": 3126, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1157, "raw_key_size": 10467, "raw_average_key_size": 23, "raw_value_size": 2644523, "raw_average_value_size": 5889, "num_data_blocks": 141, "num_entries": 449, "num_filter_entries": 449, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1725939080, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "22d0da89-8c83-45f7-a40a-8a3fa16b036a", "db_session_id": "9TWG90XKPOS2B44Z70TQ", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-10T03:31:20.356 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725939080130595, "job": 1, "event": "recovery_finished"} 2024-09-10T03:31:20.356 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-10T03:31:20.356 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-10T03:31:20.356 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi130/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-10T03:31:20.356 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5570ff6f8e00 2024-09-10T03:31:20.356 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: DB pointer 0x5570ff7fa000 2024-09-10T03:31:20.356 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-10T03:31:20.356 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-10T03:31:20.356 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: ** DB Stats ** 2024-09-10T03:31:20.357 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-10T03:31:20.357 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-10T03:31:20.357 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-10T03:31:20.357 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-10T03:31:20.357 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-10T03:31:20.357 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-10T03:31:20.357 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-10T03:31:20.357 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: 2024-09-10T03:31:20.357 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: ** Compaction Stats [default] ** 2024-09-10T03:31:20.357 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-10T03:31:20.357 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-10T03:31:20.357 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: L0 1/0 2.54 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 463.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-10T03:31:20.357 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: L6 1/0 6.80 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-10T03:31:20.357 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Sum 2/0 9.33 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 463.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-10T03:31:20.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 463.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-10T03:31:20.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: 2024-09-10T03:31:20.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: ** Compaction Stats [default] ** 2024-09-10T03:31:20.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-10T03:31:20.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-10T03:31:20.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 463.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-10T03:31:20.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: 2024-09-10T03:31:20.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-10T03:31:20.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: 2024-09-10T03:31:20.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-10T03:31:20.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-09-10T03:31:20.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-10T03:31:20.359 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-10T03:31:20.359 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-10T03:31:20.359 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-10T03:31:20.359 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Cumulative compaction: 0.00 GB write, 156.22 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-10T03:31:20.359 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Interval compaction: 0.00 GB write, 156.22 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-10T03:31:20.359 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-10T03:31:20.359 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Block cache BinnedLRUCache@0x5570ff6c9610#2 capacity: 512.00 MB usage: 4.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2.1e-05 secs_since: 0 2024-09-10T03:31:20.359 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,3.33 KB,0.000634789%) Misc(1,0.00 KB,0%) 2024-09-10T03:31:20.359 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: 2024-09-10T03:31:20.359 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-10T03:31:20.360 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: starting mon.smithi130 rank 0 at public addrs [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] at bind addrs [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi130 fsid 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:31:20.360 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: mon.smithi130@-1(???) e2 preinit fsid 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:31:20.360 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: mon.smithi130@-1(???).mds e7 new map 2024-09-10T03:31:20.360 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: mon.smithi130@-1(???).mds e7 print_map 2024-09-10T03:31:20.360 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: e7 2024-09-10T03:31:20.360 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-10T03:31:20.360 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-10T03:31:20.360 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-10T03:31:20.360 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: legacy client fscid: 1 2024-09-10T03:31:20.360 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: 2024-09-10T03:31:20.360 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Filesystem 'foofs' (1) 2024-09-10T03:31:20.360 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: fs_name foofs 2024-09-10T03:31:20.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: epoch 5 2024-09-10T03:31:20.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-10T03:31:20.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: created 2024-09-10T03:26:13.383016+0000 2024-09-10T03:31:20.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: modified 2024-09-10T03:26:19.409802+0000 2024-09-10T03:31:20.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: tableserver 0 2024-09-10T03:31:20.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: root 0 2024-09-10T03:31:20.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: session_timeout 60 2024-09-10T03:31:20.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: session_autoclose 300 2024-09-10T03:31:20.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: max_file_size 1099511627776 2024-09-10T03:31:20.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: max_xattr_size 65536 2024-09-10T03:31:20.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: required_client_features {} 2024-09-10T03:31:20.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: last_failure 0 2024-09-10T03:31:20.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: last_failure_osd_epoch 0 2024-09-10T03:31:20.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-10T03:31:20.362 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: max_mds 1 2024-09-10T03:31:20.362 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: in 0 2024-09-10T03:31:20.362 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: up {0=14494} 2024-09-10T03:31:20.362 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: failed 2024-09-10T03:31:20.362 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: damaged 2024-09-10T03:31:20.362 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: stopped 2024-09-10T03:31:20.362 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: data_pools [3] 2024-09-10T03:31:20.362 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: metadata_pool 2 2024-09-10T03:31:20.362 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: inline_data disabled 2024-09-10T03:31:20.362 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: balancer 2024-09-10T03:31:20.362 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: bal_rank_mask -1 2024-09-10T03:31:20.362 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: standby_count_wanted 1 2024-09-10T03:31:20.362 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: qdb_cluster leader: 0 members: 2024-09-10T03:31:20.362 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: [mds.foofs.smithi130.ukcxnb{0:14494} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.130:6834/1167840712,v1:172.21.15.130:6835/1167840712] compat {c=[1],r=[1],i=[7ff]}] 2024-09-10T03:31:20.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: 2024-09-10T03:31:20.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: 2024-09-10T03:31:20.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Standby daemons: 2024-09-10T03:31:20.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: 2024-09-10T03:31:20.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: [mds.foofs.smithi164.awiezo{-1:24287} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.164:6832/3151002352,v1:172.21.15.164:6833/3151002352] compat {c=[1],r=[1],i=[7ff]}] 2024-09-10T03:31:20.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: mon.smithi130@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-10T03:31:20.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: mon.smithi130@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-10T03:31:20.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: mon.smithi130@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-10T03:31:20.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: mon.smithi130@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-10T03:31:20.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 ceph-mon[63591]: mon.smithi130@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-10T03:31:20.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 bash[63572]: 22e5cf56527a49da6464cb3ff752d169e9c5add018f7978f02b85e633b6f35ce 2024-09-10T03:31:20.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:20 smithi130 systemd[1]: Started Ceph mon.smithi130 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:31:21.059 INFO:teuthology.orchestra.run.smithi130.stdout: 5 113169 11.85 MB/sec execute 150 sec latency 63.761 ms 2024-09-10T03:31:22.059 INFO:teuthology.orchestra.run.smithi130.stdout: 5 113283 11.79 MB/sec execute 151 sec latency 858.722 ms 2024-09-10T03:31:23.059 INFO:teuthology.orchestra.run.smithi130.stdout: 5 113283 11.71 MB/sec execute 152 sec latency 1858.775 ms 2024-09-10T03:31:23.416 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:23 smithi130 ceph-mon[63591]: pgmap v28: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 385 op/s 2024-09-10T03:31:23.416 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:23 smithi130 ceph-mon[63591]: pgmap v29: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.8 MiB/s wr, 610 op/s 2024-09-10T03:31:23.416 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:23 smithi130 ceph-mon[63591]: pgmap v30: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 425 op/s 2024-09-10T03:31:23.416 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:23 smithi130 ceph-mon[63591]: mon.smithi130 calling monitor election 2024-09-10T03:31:23.416 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:23 smithi130 ceph-mon[63591]: mon.smithi130 is new leader, mons smithi130,smithi164 in quorum (ranks 0,1) 2024-09-10T03:31:23.416 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:23 smithi130 ceph-mon[63591]: monmap epoch 2 2024-09-10T03:31:23.416 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:23 smithi130 ceph-mon[63591]: fsid 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:31:23.416 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:23 smithi130 ceph-mon[63591]: last_changed 2024-09-10T03:21:00.086564+0000 2024-09-10T03:31:23.417 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:23 smithi130 ceph-mon[63591]: created 2024-09-10T03:18:46.462733+0000 2024-09-10T03:31:23.417 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:23 smithi130 ceph-mon[63591]: min_mon_release 17 (quincy) 2024-09-10T03:31:23.417 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:23 smithi130 ceph-mon[63591]: election_strategy: 1 2024-09-10T03:31:23.417 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:23 smithi130 ceph-mon[63591]: 0: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.smithi130 2024-09-10T03:31:23.417 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:23 smithi130 ceph-mon[63591]: 1: [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] mon.smithi164 2024-09-10T03:31:23.417 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:23 smithi130 ceph-mon[63591]: fsmap foofs:1 {0=foofs.smithi130.ukcxnb=up:active} 1 up:standby 2024-09-10T03:31:23.417 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:23 smithi130 ceph-mon[63591]: osdmap e72: 8 total, 8 up, 8 in 2024-09-10T03:31:23.417 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:23 smithi130 ceph-mon[63591]: mgrmap e35: smithi130.axcmel(active, since 56s), standbys: smithi164.bkfwap 2024-09-10T03:31:23.417 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:23 smithi130 ceph-mon[63591]: overall HEALTH_OK 2024-09-10T03:31:23.417 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:23 smithi130 ceph-mon[63591]: from='mgr.14782 ' entity='' 2024-09-10T03:31:23.417 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:23 smithi130 ceph-mon[63591]: mgrmap e36: smithi130.axcmel(active, since 56s), standbys: smithi164.bkfwap 2024-09-10T03:31:23.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:23 smithi164 ceph-mon[25711]: pgmap v28: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 385 op/s 2024-09-10T03:31:23.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:23 smithi164 ceph-mon[25711]: pgmap v29: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.8 MiB/s wr, 610 op/s 2024-09-10T03:31:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:23 smithi164 ceph-mon[25711]: pgmap v30: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 425 op/s 2024-09-10T03:31:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:23 smithi164 ceph-mon[25711]: mon.smithi130 calling monitor election 2024-09-10T03:31:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:23 smithi164 ceph-mon[25711]: mon.smithi130 is new leader, mons smithi130,smithi164 in quorum (ranks 0,1) 2024-09-10T03:31:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:23 smithi164 ceph-mon[25711]: monmap epoch 2 2024-09-10T03:31:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:23 smithi164 ceph-mon[25711]: fsid 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:31:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:23 smithi164 ceph-mon[25711]: last_changed 2024-09-10T03:21:00.086564+0000 2024-09-10T03:31:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:23 smithi164 ceph-mon[25711]: created 2024-09-10T03:18:46.462733+0000 2024-09-10T03:31:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:23 smithi164 ceph-mon[25711]: min_mon_release 17 (quincy) 2024-09-10T03:31:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:23 smithi164 ceph-mon[25711]: election_strategy: 1 2024-09-10T03:31:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:23 smithi164 ceph-mon[25711]: 0: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.smithi130 2024-09-10T03:31:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:23 smithi164 ceph-mon[25711]: 1: [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] mon.smithi164 2024-09-10T03:31:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:23 smithi164 ceph-mon[25711]: fsmap foofs:1 {0=foofs.smithi130.ukcxnb=up:active} 1 up:standby 2024-09-10T03:31:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:23 smithi164 ceph-mon[25711]: osdmap e72: 8 total, 8 up, 8 in 2024-09-10T03:31:23.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:23 smithi164 ceph-mon[25711]: mgrmap e35: smithi130.axcmel(active, since 56s), standbys: smithi164.bkfwap 2024-09-10T03:31:23.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:23 smithi164 ceph-mon[25711]: overall HEALTH_OK 2024-09-10T03:31:23.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:23 smithi164 ceph-mon[25711]: from='mgr.14782 ' entity='' 2024-09-10T03:31:23.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:23 smithi164 ceph-mon[25711]: mgrmap e36: smithi130.axcmel(active, since 56s), standbys: smithi164.bkfwap 2024-09-10T03:31:24.059 INFO:teuthology.orchestra.run.smithi130.stdout: 5 113524 11.68 MB/sec execute 153 sec latency 1963.110 ms 2024-09-10T03:31:25.070 INFO:teuthology.orchestra.run.smithi130.stdout: 5 113777 11.65 MB/sec execute 154 sec latency 97.486 ms 2024-09-10T03:31:25.214 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:25 smithi130 ceph-mon[63591]: mgrmap e37: smithi130.axcmel(active, since 57s), standbys: smithi164.bkfwap 2024-09-10T03:31:25.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:25 smithi164 ceph-mon[25711]: mgrmap e37: smithi130.axcmel(active, since 57s), standbys: smithi164.bkfwap 2024-09-10T03:31:26.059 INFO:teuthology.orchestra.run.smithi130.stdout: 5 114136 11.60 MB/sec execute 155 sec latency 37.211 ms 2024-09-10T03:31:27.059 INFO:teuthology.orchestra.run.smithi130.stdout: 5 114388 11.57 MB/sec execute 156 sec latency 48.526 ms 2024-09-10T03:31:28.059 INFO:teuthology.orchestra.run.smithi130.stdout: 5 114617 11.54 MB/sec execute 157 sec latency 53.451 ms 2024-09-10T03:31:29.059 INFO:teuthology.orchestra.run.smithi130.stdout: 5 114938 11.51 MB/sec execute 158 sec latency 63.803 ms 2024-09-10T03:31:29.320 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:29 smithi130 ceph-mon[63591]: Standby manager daemon smithi164.bkfwap restarted 2024-09-10T03:31:29.320 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:29 smithi130 ceph-mon[63591]: Standby manager daemon smithi164.bkfwap started 2024-09-10T03:31:29.320 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:29 smithi130 ceph-mon[63591]: from='mgr.? 172.21.15.164:0/1621403174' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/crt"}]: dispatch 2024-09-10T03:31:29.320 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:29 smithi130 ceph-mon[63591]: from='mgr.? 172.21.15.164:0/1621403174' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-10T03:31:29.320 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:29 smithi130 ceph-mon[63591]: from='mgr.? 172.21.15.164:0/1621403174' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/key"}]: dispatch 2024-09-10T03:31:29.320 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:29 smithi130 ceph-mon[63591]: from='mgr.? 172.21.15.164:0/1621403174' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-10T03:31:29.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:29 smithi164 ceph-mon[25711]: Standby manager daemon smithi164.bkfwap restarted 2024-09-10T03:31:29.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:29 smithi164 ceph-mon[25711]: Standby manager daemon smithi164.bkfwap started 2024-09-10T03:31:29.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:29 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/1621403174' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/crt"}]: dispatch 2024-09-10T03:31:29.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:29 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/1621403174' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-10T03:31:29.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:29 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/1621403174' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi164.bkfwap/key"}]: dispatch 2024-09-10T03:31:29.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:29 smithi164 ceph-mon[25711]: from='mgr.? 172.21.15.164:0/1621403174' entity='mgr.smithi164.bkfwap' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-10T03:31:30.059 INFO:teuthology.orchestra.run.smithi130.stdout: 5 115331 11.48 MB/sec execute 159 sec latency 51.665 ms 2024-09-10T03:31:30.331 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: mgrmap e38: smithi130.axcmel(active, since 62s), standbys: smithi164.bkfwap 2024-09-10T03:31:30.331 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: Active manager daemon smithi130.axcmel restarted 2024-09-10T03:31:30.331 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: Activating manager daemon smithi130.axcmel 2024-09-10T03:31:30.331 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: osdmap e73: 8 total, 8 up, 8 in 2024-09-10T03:31:30.331 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: mgrmap e39: smithi130.axcmel(active, starting, since 0.00749183s), standbys: smithi164.bkfwap 2024-09-10T03:31:30.331 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mon metadata", "id": "smithi130"} : dispatch 2024-09-10T03:31:30.331 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mon metadata", "id": "smithi164"} : dispatch 2024-09-10T03:31:30.331 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds metadata", "who": "foofs.smithi130.ukcxnb"} : dispatch 2024-09-10T03:31:30.332 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds metadata", "who": "foofs.smithi164.awiezo"} : dispatch 2024-09-10T03:31:30.332 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mgr metadata", "who": "smithi130.axcmel", "id": "smithi130.axcmel"} : dispatch 2024-09-10T03:31:30.332 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mgr metadata", "who": "smithi164.bkfwap", "id": "smithi164.bkfwap"} : dispatch 2024-09-10T03:31:30.332 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-10T03:31:30.332 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-10T03:31:30.332 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-10T03:31:30.332 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-10T03:31:30.332 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-10T03:31:30.332 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-10T03:31:30.332 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-10T03:31:30.332 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-10T03:31:30.332 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds metadata"} : dispatch 2024-09-10T03:31:30.332 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata"} : dispatch 2024-09-10T03:31:30.333 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mon metadata"} : dispatch 2024-09-10T03:31:30.333 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: Manager daemon smithi130.axcmel is now available 2024-09-10T03:31:30.333 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:31:30.333 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-10T03:31:30.333 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:31:30.333 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"} : dispatch 2024-09-10T03:31:30.333 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"} : dispatch 2024-09-10T03:31:30.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: mgrmap e38: smithi130.axcmel(active, since 62s), standbys: smithi164.bkfwap 2024-09-10T03:31:30.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: Active manager daemon smithi130.axcmel restarted 2024-09-10T03:31:30.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: Activating manager daemon smithi130.axcmel 2024-09-10T03:31:30.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: osdmap e73: 8 total, 8 up, 8 in 2024-09-10T03:31:30.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: mgrmap e39: smithi130.axcmel(active, starting, since 0.00749183s), standbys: smithi164.bkfwap 2024-09-10T03:31:30.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mon metadata", "id": "smithi130"} : dispatch 2024-09-10T03:31:30.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mon metadata", "id": "smithi164"} : dispatch 2024-09-10T03:31:30.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds metadata", "who": "foofs.smithi130.ukcxnb"} : dispatch 2024-09-10T03:31:30.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds metadata", "who": "foofs.smithi164.awiezo"} : dispatch 2024-09-10T03:31:30.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mgr metadata", "who": "smithi130.axcmel", "id": "smithi130.axcmel"} : dispatch 2024-09-10T03:31:30.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mgr metadata", "who": "smithi164.bkfwap", "id": "smithi164.bkfwap"} : dispatch 2024-09-10T03:31:30.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-10T03:31:30.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-10T03:31:30.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-10T03:31:30.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-10T03:31:30.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-10T03:31:30.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-10T03:31:30.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-10T03:31:30.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-10T03:31:30.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds metadata"} : dispatch 2024-09-10T03:31:30.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata"} : dispatch 2024-09-10T03:31:30.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mon metadata"} : dispatch 2024-09-10T03:31:30.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: Manager daemon smithi130.axcmel is now available 2024-09-10T03:31:30.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:31:30.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-10T03:31:30.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:31:30.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/mirror_snapshot_schedule"} : dispatch 2024-09-10T03:31:30.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:30 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi130.axcmel/trash_purge_schedule"} : dispatch 2024-09-10T03:31:31.059 INFO:teuthology.orchestra.run.smithi130.stdout: 5 115786 11.48 MB/sec execute 160 sec latency 51.597 ms 2024-09-10T03:31:31.767 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:31 smithi164 ceph-mon[25711]: mgrmap e40: smithi130.axcmel(active, since 1.02166s), standbys: smithi164.bkfwap 2024-09-10T03:31:31.768 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:31 smithi164 ceph-mon[25711]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:31:31.768 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:31 smithi164 ceph-mon[25711]: pgmap v3: 97 pgs: 97 active+clean; 324 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-10T03:31:31.843 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:31 smithi130 ceph-mon[63591]: mgrmap e40: smithi130.axcmel(active, since 1.02166s), standbys: smithi164.bkfwap 2024-09-10T03:31:31.843 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:31 smithi130 ceph-mon[63591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-10T03:31:31.843 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:31 smithi130 ceph-mon[63591]: pgmap v3: 97 pgs: 97 active+clean; 324 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-10T03:31:32.059 INFO:teuthology.orchestra.run.smithi130.stdout: 5 116287 11.52 MB/sec execute 161 sec latency 41.114 ms 2024-09-10T03:31:32.853 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:32 smithi130 ceph-mon[63591]: [10/Sep/2024:03:31:31] ENGINE Bus STARTING 2024-09-10T03:31:32.853 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:32 smithi130 ceph-mon[63591]: pgmap v4: 97 pgs: 97 active+clean; 324 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-10T03:31:32.853 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:32 smithi130 ceph-mon[63591]: mgrmap e41: smithi130.axcmel(active, since 2s), standbys: smithi164.bkfwap 2024-09-10T03:31:32.853 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:32 smithi130 ceph-mon[63591]: [10/Sep/2024:03:31:31] ENGINE Serving on https://172.21.15.130:7150 2024-09-10T03:31:32.853 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:32 smithi130 ceph-mon[63591]: [10/Sep/2024:03:31:31] ENGINE Error in HTTPServer.serve 2024-09-10T03:31:32.853 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: Traceback (most recent call last): 2024-09-10T03:31:32.853 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-10T03:31:32.853 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: self._connections.run(self.expiration_interval) 2024-09-10T03:31:32.853 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-10T03:31:32.853 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: self._run(expiration_interval) 2024-09-10T03:31:32.854 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-10T03:31:32.854 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-10T03:31:32.854 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-10T03:31:32.854 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-10T03:31:32.854 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-10T03:31:32.854 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: s = self.context.wrap_socket( 2024-09-10T03:31:32.854 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-10T03:31:32.854 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: return self.sslsocket_class._create( 2024-09-10T03:31:32.854 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-10T03:31:32.854 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: self.do_handshake() 2024-09-10T03:31:32.854 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-10T03:31:32.854 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: self._sslobj.do_handshake() 2024-09-10T03:31:32.854 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-10T03:31:32.855 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:32 smithi130 ceph-mon[63591]: [10/Sep/2024:03:31:31] ENGINE Serving on http://172.21.15.130:8765 2024-09-10T03:31:32.855 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:32 smithi130 ceph-mon[63591]: [10/Sep/2024:03:31:31] ENGINE Bus STARTED 2024-09-10T03:31:32.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:32 smithi164 ceph-mon[25711]: [10/Sep/2024:03:31:31] ENGINE Bus STARTING 2024-09-10T03:31:32.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:32 smithi164 ceph-mon[25711]: pgmap v4: 97 pgs: 97 active+clean; 324 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-10T03:31:32.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:32 smithi164 ceph-mon[25711]: mgrmap e41: smithi130.axcmel(active, since 2s), standbys: smithi164.bkfwap 2024-09-10T03:31:32.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:32 smithi164 ceph-mon[25711]: [10/Sep/2024:03:31:31] ENGINE Serving on https://172.21.15.130:7150 2024-09-10T03:31:32.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:32 smithi164 ceph-mon[25711]: [10/Sep/2024:03:31:31] ENGINE Error in HTTPServer.serve 2024-09-10T03:31:32.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Traceback (most recent call last): 2024-09-10T03:31:32.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-10T03:31:32.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: self._connections.run(self.expiration_interval) 2024-09-10T03:31:32.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-10T03:31:32.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: self._run(expiration_interval) 2024-09-10T03:31:32.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-10T03:31:32.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-10T03:31:32.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-10T03:31:32.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-10T03:31:32.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-10T03:31:32.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: s = self.context.wrap_socket( 2024-09-10T03:31:32.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-10T03:31:32.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: return self.sslsocket_class._create( 2024-09-10T03:31:32.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-10T03:31:32.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: self.do_handshake() 2024-09-10T03:31:32.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-10T03:31:32.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: self._sslobj.do_handshake() 2024-09-10T03:31:32.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-10T03:31:32.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:32 smithi164 ceph-mon[25711]: [10/Sep/2024:03:31:31] ENGINE Serving on http://172.21.15.130:8765 2024-09-10T03:31:32.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:32 smithi164 ceph-mon[25711]: [10/Sep/2024:03:31:31] ENGINE Bus STARTED 2024-09-10T03:31:33.059 INFO:teuthology.orchestra.run.smithi130.stdout: 5 116752 11.56 MB/sec execute 162 sec latency 71.597 ms 2024-09-10T03:31:34.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 117005 11.54 MB/sec execute 163 sec latency 105.875 ms 2024-09-10T03:31:34.910 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:34 smithi130 ceph-mon[63591]: pgmap v5: 97 pgs: 97 active+clean; 324 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-10T03:31:34.910 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:34 smithi130 ceph-mon[63591]: mgrmap e42: smithi130.axcmel(active, since 4s), standbys: smithi164.bkfwap 2024-09-10T03:31:34.910 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:34 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:34.910 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:34 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:34.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:34 smithi164 ceph-mon[25711]: pgmap v5: 97 pgs: 97 active+clean; 324 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-10T03:31:34.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:34 smithi164 ceph-mon[25711]: mgrmap e42: smithi130.axcmel(active, since 4s), standbys: smithi164.bkfwap 2024-09-10T03:31:34.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:34 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:34.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:34 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:35.059 INFO:teuthology.orchestra.run.smithi130.stdout: 5 117301 11.52 MB/sec execute 164 sec latency 105.905 ms 2024-09-10T03:31:35.915 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:35 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:35.915 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:35 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:35.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:35 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:35.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:35 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:36.059 INFO:teuthology.orchestra.run.smithi130.stdout: 5 117683 11.49 MB/sec execute 165 sec latency 37.162 ms 2024-09-10T03:31:36.932 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:36 smithi130 ceph-mon[63591]: pgmap v6: 97 pgs: 97 active+clean; 324 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-10T03:31:37.041 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:36 smithi164 ceph-mon[25711]: pgmap v6: 97 pgs: 97 active+clean; 324 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-10T03:31:37.059 INFO:teuthology.orchestra.run.smithi130.stdout: 5 118059 11.48 MB/sec execute 166 sec latency 43.626 ms 2024-09-10T03:31:38.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 118390 11.46 MB/sec execute 167 sec latency 56.366 ms 2024-09-10T03:31:38.983 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:38 smithi130 ceph-mon[63591]: pgmap v7: 97 pgs: 97 active+clean; 332 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 4.7 MiB/s wr, 402 op/s 2024-09-10T03:31:38.983 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:38 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:38.983 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:38 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:39.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 118807 11.43 MB/sec execute 168 sec latency 48.311 ms 2024-09-10T03:31:39.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:38 smithi164 ceph-mon[25711]: pgmap v7: 97 pgs: 97 active+clean; 332 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 4.7 MiB/s wr, 402 op/s 2024-09-10T03:31:39.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:38 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:39.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:38 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:40.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 119202 11.40 MB/sec execute 169 sec latency 33.756 ms 2024-09-10T03:31:40.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:40 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:40.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:40 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:40.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:40 smithi130 ceph-mon[63591]: pgmap v8: 97 pgs: 97 active+clean; 332 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.6 MiB/s wr, 312 op/s 2024-09-10T03:31:40.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:40 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:40.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:40 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:40.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:40 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:31:40.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:40 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:31:40.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:40 smithi130 ceph-mon[63591]: Updating smithi130:/etc/ceph/ceph.conf 2024-09-10T03:31:40.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:40 smithi130 ceph-mon[63591]: Updating smithi164:/etc/ceph/ceph.conf 2024-09-10T03:31:40.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:40 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:40.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:40 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:40.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:40 smithi164 ceph-mon[25711]: pgmap v8: 97 pgs: 97 active+clean; 332 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.6 MiB/s wr, 312 op/s 2024-09-10T03:31:40.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:40 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:40.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:40 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:40.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:40 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:31:40.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:40 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:31:40.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:40 smithi164 ceph-mon[25711]: Updating smithi130:/etc/ceph/ceph.conf 2024-09-10T03:31:40.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:40 smithi164 ceph-mon[25711]: Updating smithi164:/etc/ceph/ceph.conf 2024-09-10T03:31:41.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 119588 11.42 MB/sec execute 170 sec latency 33.263 ms 2024-09-10T03:31:41.787 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:41 smithi130 ceph-mon[63591]: Updating smithi130:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.conf 2024-09-10T03:31:41.787 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:41 smithi130 ceph-mon[63591]: Updating smithi164:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.conf 2024-09-10T03:31:41.787 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:41 smithi130 ceph-mon[63591]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-09-10T03:31:41.787 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:41 smithi130 ceph-mon[63591]: Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2024-09-10T03:31:41.787 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:41 smithi130 ceph-mon[63591]: Updating smithi130:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.client.admin.keyring 2024-09-10T03:31:41.787 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:41 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:41.787 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:41 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:41.787 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:41 smithi130 ceph-mon[63591]: Updating smithi164:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.client.admin.keyring 2024-09-10T03:31:41.787 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:41 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:41.787 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:41 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:41.788 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:41 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:41.788 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:41 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:41.788 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:41 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:31:41.788 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:41 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:31:41.788 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:41 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "quorum_status"} : dispatch 2024-09-10T03:31:41.995 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:41 smithi164 ceph-mon[25711]: Updating smithi130:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.conf 2024-09-10T03:31:41.995 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:41 smithi164 ceph-mon[25711]: Updating smithi164:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.conf 2024-09-10T03:31:41.995 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:41 smithi164 ceph-mon[25711]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-09-10T03:31:41.995 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:41 smithi164 ceph-mon[25711]: Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2024-09-10T03:31:41.995 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:41 smithi164 ceph-mon[25711]: Updating smithi130:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.client.admin.keyring 2024-09-10T03:31:41.995 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:41 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:41.995 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:41 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:41.995 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:41 smithi164 ceph-mon[25711]: Updating smithi164:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/config/ceph.client.admin.keyring 2024-09-10T03:31:41.995 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:41 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:41.995 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:41 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:41.996 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:41 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:41.996 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:41 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:41.996 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:41 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:31:41.996 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:41 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:31:41.996 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:41 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "quorum_status"} : dispatch 2024-09-10T03:31:42.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 120042 11.45 MB/sec execute 171 sec latency 53.862 ms 2024-09-10T03:31:42.315 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:31:43.046 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:42 smithi130 ceph-mon[63591]: pgmap v9: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.3 MiB/s wr, 515 op/s 2024-09-10T03:31:43.046 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:42 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:43.046 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:42 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-10T03:31:43.046 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:42 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-10T03:31:43.046 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:42 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:31:43.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 120612 11.50 MB/sec execute 172 sec latency 38.208 ms 2024-09-10T03:31:43.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:42 smithi164 ceph-mon[25711]: pgmap v9: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.3 MiB/s wr, 515 op/s 2024-09-10T03:31:43.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:42 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:43.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:42 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-10T03:31:43.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:42 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-10T03:31:43.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:42 smithi164 ceph-mon[25711]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:31:43.529 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:31:43.529 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (2m) 4s ago 11m 20.0M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:31:43.529 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (11m) 4s ago 11m 6400k - 17.2.0 e1d6a67b021e 138ed3222ba5 2024-09-10T03:31:43.529 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (10m) 9s ago 10m 6405k - 17.2.0 e1d6a67b021e 3623793da55d 2024-09-10T03:31:43.529 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (2m) 4s ago 11m 56.2M - dad864ee21e9 72ba01ac1871 2024-09-10T03:31:43.529 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (5m) 4s ago 5m 36.8M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:31:43.529 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (5m) 9s ago 5m 15.3M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:31:43.529 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (94s) 4s ago 12m 488M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:31:43.529 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283,8765 running (42s) 9s ago 10m 451M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e4034e8e2a6d 2024-09-10T03:31:43.529 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (24s) 4s ago 12m 34.6M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 22e5cf56527a 2024-09-10T03:31:43.529 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (10m) 9s ago 10m 52.7M 2048M 17.2.0 e1d6a67b021e 7a928b0b045c 2024-09-10T03:31:43.529 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (5m) 4s ago 5m 253M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:31:43.530 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (5m) 9s ago 4m 87.3M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:31:43.530 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (2m) 4s ago 11m 10.0M - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:31:43.530 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (104s) 9s ago 10m 10.3M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:31:43.530 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (9m) 4s ago 9m 457M 1936M 17.2.0 e1d6a67b021e 1d66043abecd 2024-09-10T03:31:43.530 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (9m) 4s ago 9m 383M 1936M 17.2.0 e1d6a67b021e ca1ec04aecfa 2024-09-10T03:31:43.530 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (8m) 4s ago 8m 278M 1936M 17.2.0 e1d6a67b021e 5883f5d5e969 2024-09-10T03:31:43.530 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (8m) 4s ago 8m 415M 1936M 17.2.0 e1d6a67b021e b32084e1a404 2024-09-10T03:31:43.530 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (8m) 9s ago 8m 413M 2704M 17.2.0 e1d6a67b021e 3b5c9f00f64d 2024-09-10T03:31:43.530 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (7m) 9s ago 7m 436M 2704M 17.2.0 e1d6a67b021e f16357ac331e 2024-09-10T03:31:43.530 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (7m) 9s ago 7m 390M 2704M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:31:43.530 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (6m) 9s ago 6m 528M 2704M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:31:43.530 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (49s) 4s ago 10m 46.1M - 2.51.0 1d3b7f56885b f59b02fac5b3 2024-09-10T03:31:44.052 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:43 smithi130 ceph-mon[63591]: from='client.34128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:31:44.053 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:43 smithi130 ceph-mon[63591]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:31:44.053 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:43 smithi130 ceph-mon[63591]: Upgrade: Updating mon.smithi164 2024-09-10T03:31:44.053 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:43 smithi130 ceph-mon[63591]: Deploying daemon mon.smithi164 on smithi164 2024-09-10T03:31:44.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 121188 11.50 MB/sec execute 173 sec latency 26.840 ms 2024-09-10T03:31:44.130 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:31:44.130 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:31:44.130 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-10T03:31:44.130 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 1 2024-09-10T03:31:44.130 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:31:44.130 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:31:44.130 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:31:44.130 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:31:44.130 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:31:44.131 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-10T03:31:44.131 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:31:44.131 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:31:44.131 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:31:44.131 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:31:44.131 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:31:44.131 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-09-10T03:31:44.131 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 3 2024-09-10T03:31:44.131 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:31:44.131 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:31:44.345 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:43 smithi164 ceph-mon[25711]: from='client.34128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:31:44.346 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:43 smithi164 ceph-mon[25711]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:31:44.346 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:43 smithi164 ceph-mon[25711]: Upgrade: Updating mon.smithi164 2024-09-10T03:31:44.346 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:43 smithi164 ceph-mon[25711]: Deploying daemon mon.smithi164 on smithi164 2024-09-10T03:31:44.568 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:31:44.569 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:31:44.569 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:31:44.569 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:31:44.569 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-09-10T03:31:44.569 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-09-10T03:31:44.569 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-09-10T03:31:44.569 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "3/23 daemons upgraded", 2024-09-10T03:31:44.569 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading mon daemons", 2024-09-10T03:31:44.569 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:31:44.569 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:31:44.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:44 smithi164 systemd[1]: Stopping Ceph mon.smithi164 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:31:45.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 121515 11.49 MB/sec execute 174 sec latency 49.364 ms 2024-09-10T03:31:45.066 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:44 smithi130 ceph-mon[63591]: from='client.24617 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:31:45.066 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:44 smithi130 ceph-mon[63591]: pgmap v10: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 4.9 MiB/s wr, 471 op/s 2024-09-10T03:31:45.066 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:44 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/3540209332' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:31:45.110 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-09-10T03:31:45.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:44 smithi164 ceph-mon[25711]: from='client.24617 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:31:45.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:44 smithi164 ceph-mon[25711]: pgmap v10: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 4.9 MiB/s wr, 471 op/s 2024-09-10T03:31:45.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:44 smithi164 ceph-mon[25711]: from='client.? 172.21.15.130:0/3540209332' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:31:45.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:45 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi164[25693]: 2024-09-10T03:31:45.026+0000 7f6685547700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi164 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:31:45.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:45 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi164[25693]: 2024-09-10T03:31:45.026+0000 7f6685547700 -1 mon.smithi164@1(peon) e2 *** Got Signal Terminated *** 2024-09-10T03:31:45.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:45 smithi164 podman[49164]: 2024-09-10 03:31:45.058729755 +0000 UTC m=+0.159361343 container died 7a928b0b045c1c66777736a5fe091bdfe77eb7ab7e9b47ec99384fc8460e6318 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi164, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, vcs-type=git, name=centos-stream, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, GIT_CLEAN=True, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD) 2024-09-10T03:31:46.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 121829 11.47 MB/sec execute 175 sec latency 51.738 ms 2024-09-10T03:31:46.320 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:46 smithi164 podman[49164]: 2024-09-10 03:31:46.051530865 +0000 UTC m=+1.152162459 container cleanup 7a928b0b045c1c66777736a5fe091bdfe77eb7ab7e9b47ec99384fc8460e6318 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi164, CEPH_POINT_RELEASE=-17.2.0, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, distribution-scope=public, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, name=centos-stream, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, io.openshift.expose-services=, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-10T03:31:46.321 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:46 smithi164 bash[49164]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi164 2024-09-10T03:31:46.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:46 smithi164 podman[49175]: 2024-09-10 03:31:46.320237898 +0000 UTC m=+1.264687596 container remove 7a928b0b045c1c66777736a5fe091bdfe77eb7ab7e9b47ec99384fc8460e6318 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi164, maintainer=Guillaume Abrioux , io.openshift.expose-services=, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, architecture=x86_64, version=8, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, name=centos-stream, distribution-scope=public) 2024-09-10T03:31:46.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:46 smithi164 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@mon.smithi164.service: Deactivated successfully. 2024-09-10T03:31:46.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:46 smithi164 systemd[1]: Stopped Ceph mon.smithi164 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:31:46.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:46 smithi164 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@mon.smithi164.service: Consumed 7.264s CPU time. 2024-09-10T03:31:47.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 122195 11.43 MB/sec execute 176 sec latency 67.321 ms 2024-09-10T03:31:47.141 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:46 smithi164 systemd[1]: Starting Ceph mon.smithi164 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:31:47.141 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 podman[49299]: 2024-09-10 03:31:46.989311419 +0000 UTC m=+0.025088857 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:31:47.425 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 podman[49299]: 2024-09-10 03:31:47.153735747 +0000 UTC m=+0.189513176 container create 7d80bd0d15b99065d568954cf658e5c67afe9ad8bd6aa29c7a958be0168d0b34 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi164, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-10T03:31:47.425 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 podman[49299]: 2024-09-10 03:31:47.425329144 +0000 UTC m=+0.461106567 container init 7d80bd0d15b99065d568954cf658e5c67afe9ad8bd6aa29c7a958be0168d0b34 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi164, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-10T03:31:47.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 podman[49299]: 2024-09-10 03:31:47.428613612 +0000 UTC m=+0.464391037 container start 7d80bd0d15b99065d568954cf658e5c67afe9ad8bd6aa29c7a958be0168d0b34 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi164, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-10T03:31:47.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: set uid:gid to 167:167 (ceph:ceph) 2024-09-10T03:31:47.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev), process ceph-mon, pid 2 2024-09-10T03:31:47.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: pidfile_write: ignore empty --pid-file 2024-09-10T03:31:47.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: load: jerasure load: lrc 2024-09-10T03:31:47.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: RocksDB version: 7.9.2 2024-09-10T03:31:47.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Git sha 0 2024-09-10T03:31:47.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Compile date 2024-09-10 00:49:01 2024-09-10T03:31:47.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: DB SUMMARY 2024-09-10T03:31:47.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: DB Session ID: 0OX8SRORR2LOT0EE9H4X 2024-09-10T03:31:47.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: CURRENT file: CURRENT 2024-09-10T03:31:47.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: IDENTITY file: IDENTITY 2024-09-10T03:31:47.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: MANIFEST file: MANIFEST-000009 size: 903 Bytes 2024-09-10T03:31:47.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi164/store.db dir, Total Num: 1, files: 000024.sst 2024-09-10T03:31:47.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi164/store.db: 000022.log size: 7813041 ; 2024-09-10T03:31:47.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.error_if_exists: 0 2024-09-10T03:31:47.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.create_if_missing: 0 2024-09-10T03:31:47.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.paranoid_checks: 1 2024-09-10T03:31:47.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-10T03:31:47.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-10T03:31:47.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-10T03:31:47.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.env: 0x558e3b453120 2024-09-10T03:31:47.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.fs: PosixFileSystem 2024-09-10T03:31:47.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.info_log: 0x558e3d0bb8e0 2024-09-10T03:31:47.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_file_opening_threads: 16 2024-09-10T03:31:47.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.statistics: (nil) 2024-09-10T03:31:47.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.use_fsync: 0 2024-09-10T03:31:47.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_log_file_size: 0 2024-09-10T03:31:47.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-10T03:31:47.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-10T03:31:47.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.keep_log_file_num: 1000 2024-09-10T03:31:47.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.recycle_log_file_num: 0 2024-09-10T03:31:47.710 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.allow_fallocate: 1 2024-09-10T03:31:47.711 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.allow_mmap_reads: 0 2024-09-10T03:31:47.712 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.allow_mmap_writes: 0 2024-09-10T03:31:47.712 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.use_direct_reads: 0 2024-09-10T03:31:47.712 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-10T03:31:47.712 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.create_missing_column_families: 0 2024-09-10T03:31:47.712 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.db_log_dir: 2024-09-10T03:31:47.712 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.wal_dir: 2024-09-10T03:31:47.712 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-10T03:31:47.712 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-10T03:31:47.712 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-10T03:31:47.712 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-10T03:31:47.712 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-10T03:31:47.712 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-10T03:31:47.712 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.advise_random_on_open: 1 2024-09-10T03:31:47.713 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.db_write_buffer_size: 0 2024-09-10T03:31:47.713 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.write_buffer_manager: 0x558e3d0bfb80 2024-09-10T03:31:47.713 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-10T03:31:47.713 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-10T03:31:47.713 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-10T03:31:47.713 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.rate_limiter: (nil) 2024-09-10T03:31:47.713 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-10T03:31:47.713 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.wal_recovery_mode: 2 2024-09-10T03:31:47.713 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.enable_thread_tracking: 0 2024-09-10T03:31:47.713 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.enable_pipelined_write: 0 2024-09-10T03:31:47.713 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.unordered_write: 0 2024-09-10T03:31:47.713 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-10T03:31:47.713 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-10T03:31:47.713 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-10T03:31:47.714 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-10T03:31:47.714 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.row_cache: None 2024-09-10T03:31:47.714 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.wal_filter: None 2024-09-10T03:31:47.714 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-10T03:31:47.714 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.allow_ingest_behind: 0 2024-09-10T03:31:47.714 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.two_write_queues: 0 2024-09-10T03:31:47.714 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.manual_wal_flush: 0 2024-09-10T03:31:47.714 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.wal_compression: 0 2024-09-10T03:31:47.714 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.atomic_flush: 0 2024-09-10T03:31:47.714 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-10T03:31:47.714 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-10T03:31:47.714 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-10T03:31:47.714 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.log_readahead_size: 0 2024-09-10T03:31:47.714 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-10T03:31:47.714 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.best_efforts_recovery: 0 2024-09-10T03:31:47.715 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-10T03:31:47.715 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-10T03:31:47.715 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.allow_data_in_errors: 0 2024-09-10T03:31:47.715 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.db_host_id: __hostname__ 2024-09-10T03:31:47.715 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-10T03:31:47.715 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_background_jobs: 2 2024-09-10T03:31:47.715 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_background_compactions: -1 2024-09-10T03:31:47.715 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_subcompactions: 1 2024-09-10T03:31:47.715 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-10T03:31:47.715 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-10T03:31:47.715 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-10T03:31:47.715 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_total_wal_size: 0 2024-09-10T03:31:47.715 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-10T03:31:47.715 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-10T03:31:47.716 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-10T03:31:47.716 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-10T03:31:47.716 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_open_files: -1 2024-09-10T03:31:47.716 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.bytes_per_sync: 0 2024-09-10T03:31:47.716 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-10T03:31:47.716 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-10T03:31:47.716 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compaction_readahead_size: 0 2024-09-10T03:31:47.716 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_background_flushes: -1 2024-09-10T03:31:47.716 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Compression algorithms supported: 2024-09-10T03:31:47.716 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: kZSTD supported: 0 2024-09-10T03:31:47.716 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: kXpressCompression supported: 0 2024-09-10T03:31:47.716 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: kBZip2Compression supported: 0 2024-09-10T03:31:47.716 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-10T03:31:47.716 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: kLZ4Compression supported: 1 2024-09-10T03:31:47.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: kZlibCompression supported: 1 2024-09-10T03:31:47.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: kLZ4HCCompression supported: 1 2024-09-10T03:31:47.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: kSnappyCompression supported: 1 2024-09-10T03:31:47.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-10T03:31:47.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-10T03:31:47.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi164/store.db/MANIFEST-000009 2024-09-10T03:31:47.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-10T03:31:47.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-10T03:31:47.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.merge_operator: 2024-09-10T03:31:47.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compaction_filter: None 2024-09-10T03:31:47.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compaction_filter_factory: None 2024-09-10T03:31:47.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.sst_partitioner_factory: None 2024-09-10T03:31:47.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-10T03:31:47.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-10T03:31:47.717 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558e3d0bb540) 2024-09-10T03:31:47.718 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: cache_index_and_filter_blocks: 1 2024-09-10T03:31:47.718 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-10T03:31:47.718 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-10T03:31:47.718 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: pin_top_level_index_and_filter: 1 2024-09-10T03:31:47.718 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: index_type: 0 2024-09-10T03:31:47.718 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: data_block_index_type: 0 2024-09-10T03:31:47.718 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: index_shortening: 1 2024-09-10T03:31:47.718 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-10T03:31:47.718 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: checksum: 4 2024-09-10T03:31:47.718 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: no_block_cache: 0 2024-09-10T03:31:47.718 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_cache: 0x558e3d0af610 2024-09-10T03:31:47.718 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_cache_name: BinnedLRUCache 2024-09-10T03:31:47.718 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_cache_options: 2024-09-10T03:31:47.718 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: capacity : 536870912 2024-09-10T03:31:47.719 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: num_shard_bits : 4 2024-09-10T03:31:47.719 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: strict_capacity_limit : 0 2024-09-10T03:31:47.719 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: high_pri_pool_ratio: 0.000 2024-09-10T03:31:47.719 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_cache_compressed: (nil) 2024-09-10T03:31:47.719 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: persistent_cache: (nil) 2024-09-10T03:31:47.719 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_size: 4096 2024-09-10T03:31:47.719 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_size_deviation: 10 2024-09-10T03:31:47.719 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_restart_interval: 16 2024-09-10T03:31:47.719 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: index_block_restart_interval: 1 2024-09-10T03:31:47.719 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: metadata_block_size: 4096 2024-09-10T03:31:47.719 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: partition_filters: 0 2024-09-10T03:31:47.719 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: use_delta_encoding: 1 2024-09-10T03:31:47.719 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: filter_policy: bloomfilter 2024-09-10T03:31:47.719 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: whole_key_filtering: 1 2024-09-10T03:31:47.719 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: verify_compression: 0 2024-09-10T03:31:47.720 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: read_amp_bytes_per_bit: 0 2024-09-10T03:31:47.720 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: format_version: 5 2024-09-10T03:31:47.720 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: enable_index_compression: 1 2024-09-10T03:31:47.720 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: block_align: 0 2024-09-10T03:31:47.720 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: max_auto_readahead_size: 262144 2024-09-10T03:31:47.720 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: prepopulate_block_cache: 0 2024-09-10T03:31:47.720 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: initial_auto_readahead_size: 8192 2024-09-10T03:31:47.720 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: num_file_reads_for_auto_readahead: 2 2024-09-10T03:31:47.720 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.write_buffer_size: 33554432 2024-09-10T03:31:47.720 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_write_buffer_number: 2 2024-09-10T03:31:47.720 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compression: NoCompression 2024-09-10T03:31:47.720 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.bottommost_compression: Disabled 2024-09-10T03:31:47.720 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.prefix_extractor: nullptr 2024-09-10T03:31:47.720 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-10T03:31:47.721 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.num_levels: 7 2024-09-10T03:31:47.721 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-10T03:31:47.721 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-10T03:31:47.721 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-10T03:31:47.721 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-10T03:31:47.721 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-10T03:31:47.721 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-10T03:31:47.721 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-10T03:31:47.721 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-10T03:31:47.721 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-10T03:31:47.721 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-10T03:31:47.721 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-10T03:31:47.721 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-10T03:31:47.721 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-10T03:31:47.722 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compression_opts.level: 32767 2024-09-10T03:31:47.722 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compression_opts.strategy: 0 2024-09-10T03:31:47.722 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-10T03:31:47.722 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-10T03:31:47.722 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-10T03:31:47.722 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-10T03:31:47.722 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compression_opts.enabled: false 2024-09-10T03:31:47.722 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-10T03:31:47.722 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-10T03:31:47.722 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-10T03:31:47.722 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-10T03:31:47.722 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.target_file_size_base: 67108864 2024-09-10T03:31:47.722 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-10T03:31:47.722 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-10T03:31:47.722 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-10T03:31:47.723 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-10T03:31:47.723 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-10T03:31:47.723 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-10T03:31:47.723 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-10T03:31:47.723 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-10T03:31:47.723 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-10T03:31:47.723 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-10T03:31:47.723 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-10T03:31:47.723 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-10T03:31:47.723 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-10T03:31:47.723 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-10T03:31:47.723 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.arena_block_size: 1048576 2024-09-10T03:31:47.723 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-10T03:31:47.723 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-10T03:31:47.724 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.disable_auto_compactions: 0 2024-09-10T03:31:47.724 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-10T03:31:47.724 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-10T03:31:47.724 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-10T03:31:47.724 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-10T03:31:47.724 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-10T03:31:47.724 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-10T03:31:47.724 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-10T03:31:47.724 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-10T03:31:47.724 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-10T03:31:47.724 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-10T03:31:47.724 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-10T03:31:47.724 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.inplace_update_support: 0 2024-09-10T03:31:47.724 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-10T03:31:47.725 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-10T03:31:47.725 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-10T03:31:47.725 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-10T03:31:47.725 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.bloom_locality: 0 2024-09-10T03:31:47.725 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.max_successive_merges: 0 2024-09-10T03:31:47.725 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-10T03:31:47.725 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.paranoid_file_checks: 0 2024-09-10T03:31:47.725 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.force_consistency_checks: 1 2024-09-10T03:31:47.725 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.report_bg_io_stats: 0 2024-09-10T03:31:47.725 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.ttl: 2592000 2024-09-10T03:31:47.725 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-10T03:31:47.725 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-10T03:31:47.725 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-10T03:31:47.725 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.enable_blob_files: false 2024-09-10T03:31:47.726 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.min_blob_size: 0 2024-09-10T03:31:47.726 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.blob_file_size: 268435456 2024-09-10T03:31:47.726 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-10T03:31:47.726 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-10T03:31:47.726 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-10T03:31:47.726 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-10T03:31:47.726 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-10T03:31:47.726 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.blob_file_starting_level: 0 2024-09-10T03:31:47.726 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-10T03:31:47.726 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-09-10T03:31:47.726 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-10T03:31:47.726 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi164/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12233, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-10T03:31:47.726 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-09-10T03:31:47.726 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 636c0bd4-9992-4abe-9479-5b4e9b327fd2 2024-09-10T03:31:47.727 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725939107456700, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-09-10T03:31:47.727 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-09-10T03:31:47.727 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725939107474504, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 4779845, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12238, "largest_seqno": 13885, "table_properties": {"data_size": 4772297, "index_size": 4775, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1733, "raw_key_size": 15646, "raw_average_key_size": 23, "raw_value_size": 4757753, "raw_average_value_size": 7090, "num_data_blocks": 215, "num_entries": 671, "num_filter_entries": 671, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1725939107, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "636c0bd4-9992-4abe-9479-5b4e9b327fd2", "db_session_id": "0OX8SRORR2LOT0EE9H4X", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-09-10T03:31:47.727 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725939107474637, "job": 1, "event": "recovery_finished"} 2024-09-10T03:31:47.727 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-09-10T03:31:47.727 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-10T03:31:47.727 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi164/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-10T03:31:47.727 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x558e3d0dee00 2024-09-10T03:31:47.727 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: DB pointer 0x558e3d1e0000 2024-09-10T03:31:47.727 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-10T03:31:47.727 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-10T03:31:47.727 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ** DB Stats ** 2024-09-10T03:31:47.727 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-10T03:31:47.727 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-10T03:31:47.728 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-10T03:31:47.728 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-10T03:31:47.728 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-10T03:31:47.728 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-10T03:31:47.728 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-10T03:31:47.728 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-09-10T03:31:47.728 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ** Compaction Stats [default] ** 2024-09-10T03:31:47.728 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-10T03:31:47.728 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-10T03:31:47.728 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: L0 1/0 4.56 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 606.2 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-10T03:31:47.728 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: L6 1/0 6.80 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-10T03:31:47.728 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Sum 2/0 11.36 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 606.2 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-10T03:31:47.728 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 606.2 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-10T03:31:47.728 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-09-10T03:31:47.729 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ** Compaction Stats [default] ** 2024-09-10T03:31:47.729 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-10T03:31:47.729 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-10T03:31:47.729 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 606.2 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-10T03:31:47.729 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-09-10T03:31:47.729 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-10T03:31:47.729 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-09-10T03:31:47.729 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-10T03:31:47.729 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-09-10T03:31:47.729 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-10T03:31:47.729 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-10T03:31:47.729 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-10T03:31:47.729 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-10T03:31:47.729 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Cumulative compaction: 0.00 GB write, 187.27 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-10T03:31:47.730 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Interval compaction: 0.00 GB write, 187.27 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-10T03:31:47.730 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-10T03:31:47.730 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Block cache BinnedLRUCache@0x558e3d0af610#2 capacity: 512.00 MB usage: 6.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-10T03:31:47.730 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-09-10T03:31:47.730 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-09-10T03:31:47.730 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-10T03:31:47.730 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: starting mon.smithi164 rank 1 at public addrs [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] at bind addrs [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi164 fsid 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:31:47.730 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: mon.smithi164@-1(???) e2 preinit fsid 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:31:47.730 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: mon.smithi164@-1(???).mds e7 new map 2024-09-10T03:31:47.730 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: mon.smithi164@-1(???).mds e7 print_map 2024-09-10T03:31:47.730 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: e7 2024-09-10T03:31:47.730 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-10T03:31:47.730 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-10T03:31:47.730 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-10T03:31:47.730 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: legacy client fscid: 1 2024-09-10T03:31:47.731 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-09-10T03:31:47.731 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Filesystem 'foofs' (1) 2024-09-10T03:31:47.731 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: fs_name foofs 2024-09-10T03:31:47.731 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: epoch 5 2024-09-10T03:31:47.731 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-10T03:31:47.731 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: created 2024-09-10T03:26:13.383016+0000 2024-09-10T03:31:47.731 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: modified 2024-09-10T03:26:19.409802+0000 2024-09-10T03:31:47.731 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: tableserver 0 2024-09-10T03:31:47.731 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: root 0 2024-09-10T03:31:47.731 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: session_timeout 60 2024-09-10T03:31:47.731 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: session_autoclose 300 2024-09-10T03:31:47.731 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: max_file_size 1099511627776 2024-09-10T03:31:47.731 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: max_xattr_size 65536 2024-09-10T03:31:47.731 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: required_client_features {} 2024-09-10T03:31:47.732 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: last_failure 0 2024-09-10T03:31:47.732 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: last_failure_osd_epoch 0 2024-09-10T03:31:47.732 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-10T03:31:47.732 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: max_mds 1 2024-09-10T03:31:47.732 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: in 0 2024-09-10T03:31:47.732 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: up {0=14494} 2024-09-10T03:31:47.732 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: failed 2024-09-10T03:31:47.732 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: damaged 2024-09-10T03:31:47.732 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: stopped 2024-09-10T03:31:47.732 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: data_pools [3] 2024-09-10T03:31:47.732 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: metadata_pool 2 2024-09-10T03:31:47.732 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: inline_data disabled 2024-09-10T03:31:47.732 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: balancer 2024-09-10T03:31:47.732 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: bal_rank_mask -1 2024-09-10T03:31:47.733 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: standby_count_wanted 1 2024-09-10T03:31:47.733 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: qdb_cluster leader: 0 members: 2024-09-10T03:31:47.733 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: [mds.foofs.smithi130.ukcxnb{0:14494} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.130:6834/1167840712,v1:172.21.15.130:6835/1167840712] compat {c=[1],r=[1],i=[7ff]}] 2024-09-10T03:31:47.733 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-09-10T03:31:47.733 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-09-10T03:31:47.733 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: Standby daemons: 2024-09-10T03:31:47.733 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: 2024-09-10T03:31:47.733 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout: [mds.foofs.smithi164.awiezo{-1:24287} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.164:6832/3151002352,v1:172.21.15.164:6833/3151002352] compat {c=[1],r=[1],i=[7ff]}] 2024-09-10T03:31:47.733 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: mon.smithi164@-1(???).osd e73 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-10T03:31:47.733 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: mon.smithi164@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-10T03:31:47.733 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: mon.smithi164@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-10T03:31:47.733 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: mon.smithi164@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-10T03:31:47.733 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 ceph-mon[49318]: mon.smithi164@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-09-10T03:31:47.733 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 bash[49299]: 7d80bd0d15b99065d568954cf658e5c67afe9ad8bd6aa29c7a958be0168d0b34 2024-09-10T03:31:47.734 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:47 smithi164 systemd[1]: Started Ceph mon.smithi164 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:31:48.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 122582 11.41 MB/sec execute 177 sec latency 75.576 ms 2024-09-10T03:31:49.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 122913 11.40 MB/sec execute 178 sec latency 35.352 ms 2024-09-10T03:31:49.428 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: pgmap v12: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 7.9 MiB/s wr, 746 op/s 2024-09-10T03:31:49.428 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mon metadata", "id": "smithi130"} : dispatch 2024-09-10T03:31:49.428 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: mon.smithi130 calling monitor election 2024-09-10T03:31:49.428 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: mon.smithi164 calling monitor election 2024-09-10T03:31:49.428 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mon metadata", "id": "smithi164"} : dispatch 2024-09-10T03:31:49.428 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: mon.smithi130 is new leader, mons smithi130,smithi164 in quorum (ranks 0,1) 2024-09-10T03:31:49.428 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: monmap epoch 3 2024-09-10T03:31:49.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: fsid 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:31:49.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: last_changed 2024-09-10T03:31:48.180965+0000 2024-09-10T03:31:49.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: created 2024-09-10T03:18:46.462733+0000 2024-09-10T03:31:49.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: min_mon_release 19 (squid) 2024-09-10T03:31:49.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: election_strategy: 1 2024-09-10T03:31:49.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: 0: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.smithi130 2024-09-10T03:31:49.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: 1: [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] mon.smithi164 2024-09-10T03:31:49.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: fsmap foofs:1 {0=foofs.smithi130.ukcxnb=up:active} 1 up:standby 2024-09-10T03:31:49.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: osdmap e73: 8 total, 8 up, 8 in 2024-09-10T03:31:49.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: mgrmap e42: smithi130.axcmel(active, since 18s), standbys: smithi164.bkfwap 2024-09-10T03:31:49.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: overall HEALTH_OK 2024-09-10T03:31:49.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:49.429 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:49.430 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:49 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:31:49.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: pgmap v12: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 7.9 MiB/s wr, 746 op/s 2024-09-10T03:31:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mon metadata", "id": "smithi130"} : dispatch 2024-09-10T03:31:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: mon.smithi130 calling monitor election 2024-09-10T03:31:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: mon.smithi164 calling monitor election 2024-09-10T03:31:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mon metadata", "id": "smithi164"} : dispatch 2024-09-10T03:31:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: mon.smithi130 is new leader, mons smithi130,smithi164 in quorum (ranks 0,1) 2024-09-10T03:31:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: monmap epoch 3 2024-09-10T03:31:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: fsid 5829498e-6f23-11ef-bcea-c7b262605968 2024-09-10T03:31:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: last_changed 2024-09-10T03:31:48.180965+0000 2024-09-10T03:31:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: created 2024-09-10T03:18:46.462733+0000 2024-09-10T03:31:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: min_mon_release 19 (squid) 2024-09-10T03:31:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: election_strategy: 1 2024-09-10T03:31:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: 0: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.smithi130 2024-09-10T03:31:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: 1: [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] mon.smithi164 2024-09-10T03:31:49.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: fsmap foofs:1 {0=foofs.smithi130.ukcxnb=up:active} 1 up:standby 2024-09-10T03:31:49.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: osdmap e73: 8 total, 8 up, 8 in 2024-09-10T03:31:49.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: mgrmap e42: smithi130.axcmel(active, since 18s), standbys: smithi164.bkfwap 2024-09-10T03:31:49.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: overall HEALTH_OK 2024-09-10T03:31:49.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:49.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:49.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:49 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:31:50.061 INFO:teuthology.orchestra.run.smithi130.stdout: 5 123254 11.41 MB/sec execute 179 sec latency 58.537 ms 2024-09-10T03:31:51.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 123667 11.42 MB/sec execute 180 sec latency 54.431 ms 2024-09-10T03:31:51.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:51 smithi130 ceph-mon[63591]: pgmap v13: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 512 op/s 2024-09-10T03:31:51.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:51 smithi164 ceph-mon[49318]: pgmap v13: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 512 op/s 2024-09-10T03:31:52.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 124013 11.43 MB/sec execute 181 sec latency 46.458 ms 2024-09-10T03:31:53.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 124302 11.42 MB/sec execute 182 sec latency 66.988 ms 2024-09-10T03:31:53.334 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:53 smithi130 ceph-mon[63591]: pgmap v14: 97 pgs: 97 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 729 op/s 2024-09-10T03:31:53.335 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:53 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:53.335 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:53 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:53.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:53 smithi164 ceph-mon[49318]: pgmap v14: 97 pgs: 97 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 729 op/s 2024-09-10T03:31:53.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:53 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:53.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:53 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:54.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 124607 11.39 MB/sec execute 183 sec latency 40.653 ms 2024-09-10T03:31:54.594 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:54 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:54.595 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:54 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:54.595 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:54 smithi130 ceph-mon[63591]: pgmap v15: 97 pgs: 97 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 491 op/s 2024-09-10T03:31:54.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:54 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:54.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:54 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:54.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:54 smithi164 ceph-mon[49318]: pgmap v15: 97 pgs: 97 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 491 op/s 2024-09-10T03:31:55.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 125091 11.39 MB/sec execute 184 sec latency 25.151 ms 2024-09-10T03:31:56.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 125549 11.38 MB/sec execute 185 sec latency 40.892 ms 2024-09-10T03:31:56.871 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:56 smithi130 ceph-mon[63591]: pgmap v16: 97 pgs: 97 active+clean; 332 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.0 MiB/s wr, 493 op/s 2024-09-10T03:31:57.058 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:56 smithi164 ceph-mon[49318]: pgmap v16: 97 pgs: 97 active+clean; 332 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.0 MiB/s wr, 493 op/s 2024-09-10T03:31:57.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 125911 11.35 MB/sec execute 186 sec latency 75.410 ms 2024-09-10T03:31:58.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 126247 11.32 MB/sec execute 187 sec latency 64.479 ms 2024-09-10T03:31:58.158 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:58 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:58.158 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:58 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:58.158 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:58 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:31:58.158 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:58 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:31:58.158 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:58 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:58.158 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:58 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:58.158 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:58 smithi130 ceph-mon[63591]: Reconfiguring mon.smithi130 (monmap changed)... 2024-09-10T03:31:58.158 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:58 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-10T03:31:58.158 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:58 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-10T03:31:58.159 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:58 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:31:58.159 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:58 smithi130 ceph-mon[63591]: Reconfiguring daemon mon.smithi130 on smithi130 2024-09-10T03:31:58.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:58 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:58.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:58 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:58.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:58 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:31:58.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:58 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:31:58.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:58 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:58.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:58 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:31:58.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:58 smithi164 ceph-mon[49318]: Reconfiguring mon.smithi130 (monmap changed)... 2024-09-10T03:31:58.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:58 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-10T03:31:58.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:58 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-10T03:31:58.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:58 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:31:58.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:58 smithi164 ceph-mon[49318]: Reconfiguring daemon mon.smithi130 on smithi130 2024-09-10T03:31:59.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 126522 11.30 MB/sec execute 188 sec latency 53.431 ms 2024-09-10T03:31:59.181 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:31:59 smithi130 ceph-mon[63591]: pgmap v17: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 7.6 MiB/s wr, 724 op/s 2024-09-10T03:31:59.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:31:59 smithi164 ceph-mon[49318]: pgmap v17: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 7.6 MiB/s wr, 724 op/s 2024-09-10T03:32:00.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 126977 11.34 MB/sec execute 189 sec latency 46.339 ms 2024-09-10T03:32:00.708 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:00.708 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:00.709 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:00 smithi130 ceph-mon[63591]: Reconfiguring mgr.smithi130.axcmel (monmap changed)... 2024-09-10T03:32:00.709 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi130.axcmel", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-10T03:32:00.709 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mgr services"} : dispatch 2024-09-10T03:32:00.709 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:00.709 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:00 smithi130 ceph-mon[63591]: Reconfiguring daemon mgr.smithi130.axcmel on smithi130 2024-09-10T03:32:00.709 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:00 smithi130 ceph-mon[63591]: pgmap v18: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 450 op/s 2024-09-10T03:32:00.709 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:32:00.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:00 smithi164 ceph-mon[49318]: Reconfiguring mgr.smithi130.axcmel (monmap changed)... 2024-09-10T03:32:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi130.axcmel", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-10T03:32:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mgr services"} : dispatch 2024-09-10T03:32:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:00 smithi164 ceph-mon[49318]: Reconfiguring daemon mgr.smithi130.axcmel on smithi130 2024-09-10T03:32:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:00 smithi164 ceph-mon[49318]: pgmap v18: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 450 op/s 2024-09-10T03:32:00.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:32:01.060 INFO:teuthology.orchestra.run.smithi130.stdout: 5 127532 11.38 MB/sec execute 190 sec latency 30.783 ms 2024-09-10T03:32:02.061 INFO:teuthology.orchestra.run.smithi130.stdout: 5 127899 11.39 MB/sec execute 191 sec latency 36.096 ms 2024-09-10T03:32:02.972 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:02 smithi130 ceph-mon[63591]: pgmap v19: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 679 op/s 2024-09-10T03:32:02.972 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:02 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:02.972 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:02 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:02.972 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:02 smithi130 ceph-mon[63591]: Reconfiguring crash.smithi130 (monmap changed)... 2024-09-10T03:32:02.972 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:02 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi130", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-10T03:32:02.972 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:02 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:02.972 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:02 smithi130 ceph-mon[63591]: Reconfiguring daemon crash.smithi130 on smithi130 2024-09-10T03:32:03.061 INFO:teuthology.orchestra.run.smithi130.stdout: 5 128386 11.38 MB/sec execute 192 sec latency 25.258 ms 2024-09-10T03:32:03.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:02 smithi164 ceph-mon[49318]: pgmap v19: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 679 op/s 2024-09-10T03:32:03.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:02 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:03.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:02 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:03.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:02 smithi164 ceph-mon[49318]: Reconfiguring crash.smithi130 (monmap changed)... 2024-09-10T03:32:03.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:02 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi130", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-10T03:32:03.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:02 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:03.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:02 smithi164 ceph-mon[49318]: Reconfiguring daemon crash.smithi130 on smithi130 2024-09-10T03:32:04.061 INFO:teuthology.orchestra.run.smithi130.stdout: 5 128820 11.37 MB/sec execute 193 sec latency 40.862 ms 2024-09-10T03:32:05.057 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:04 smithi130 ceph-mon[63591]: pgmap v20: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 463 op/s 2024-09-10T03:32:05.057 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:04 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:05.057 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:04 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:05.057 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:04 smithi130 ceph-mon[63591]: Reconfiguring osd.0 (monmap changed)... 2024-09-10T03:32:05.057 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:04 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-10T03:32:05.057 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:04 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:05.057 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:04 smithi130 ceph-mon[63591]: Reconfiguring daemon osd.0 on smithi130 2024-09-10T03:32:05.061 INFO:teuthology.orchestra.run.smithi130.stdout: 5 129106 11.35 MB/sec execute 194 sec latency 54.373 ms 2024-09-10T03:32:05.282 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:04 smithi164 ceph-mon[49318]: pgmap v20: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 463 op/s 2024-09-10T03:32:05.282 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:04 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:05.282 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:04 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:05.282 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:04 smithi164 ceph-mon[49318]: Reconfiguring osd.0 (monmap changed)... 2024-09-10T03:32:05.282 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:04 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-10T03:32:05.283 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:04 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:05.283 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:04 smithi164 ceph-mon[49318]: Reconfiguring daemon osd.0 on smithi130 2024-09-10T03:32:06.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 129398 11.32 MB/sec execute 195 sec latency 88.812 ms 2024-09-10T03:32:07.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 129733 11.28 MB/sec execute 196 sec latency 61.360 ms 2024-09-10T03:32:07.128 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:06 smithi130 ceph-mon[63591]: pgmap v21: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.9 MiB/s wr, 464 op/s 2024-09-10T03:32:07.129 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:06 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:07.129 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:06 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:07.129 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:06 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-10T03:32:07.129 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:06 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:07.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:06 smithi164 ceph-mon[49318]: pgmap v21: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.9 MiB/s wr, 464 op/s 2024-09-10T03:32:07.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:06 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:07.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:06 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:07.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:06 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-10T03:32:07.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:06 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:08.061 INFO:teuthology.orchestra.run.smithi130.stdout: 5 129981 11.27 MB/sec execute 197 sec latency 63.322 ms 2024-09-10T03:32:08.202 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:07 smithi130 ceph-mon[63591]: Reconfiguring osd.1 (monmap changed)... 2024-09-10T03:32:08.202 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:07 smithi130 ceph-mon[63591]: Reconfiguring daemon osd.1 on smithi130 2024-09-10T03:32:08.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:07 smithi164 ceph-mon[49318]: Reconfiguring osd.1 (monmap changed)... 2024-09-10T03:32:08.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:07 smithi164 ceph-mon[49318]: Reconfiguring daemon osd.1 on smithi130 2024-09-10T03:32:09.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 130460 11.30 MB/sec execute 198 sec latency 35.704 ms 2024-09-10T03:32:09.219 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:08 smithi130 ceph-mon[63591]: pgmap v22: 97 pgs: 97 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.2 MiB/s wr, 706 op/s 2024-09-10T03:32:09.219 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:08 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:09.219 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:08 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:09.219 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:08 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-10T03:32:09.219 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:08 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:09.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:08 smithi164 ceph-mon[49318]: pgmap v22: 97 pgs: 97 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.2 MiB/s wr, 706 op/s 2024-09-10T03:32:09.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:08 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:09.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:08 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:09.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:08 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-10T03:32:09.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:08 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:10.007 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:09 smithi130 ceph-mon[63591]: Reconfiguring osd.2 (monmap changed)... 2024-09-10T03:32:10.008 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:09 smithi130 ceph-mon[63591]: Reconfiguring daemon osd.2 on smithi130 2024-09-10T03:32:10.061 INFO:teuthology.orchestra.run.smithi130.stdout: 5 130925 11.31 MB/sec execute 199 sec latency 51.097 ms 2024-09-10T03:32:10.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:09 smithi164 ceph-mon[49318]: Reconfiguring osd.2 (monmap changed)... 2024-09-10T03:32:10.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:09 smithi164 ceph-mon[49318]: Reconfiguring daemon osd.2 on smithi130 2024-09-10T03:32:11.018 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:10 smithi130 ceph-mon[63591]: pgmap v23: 97 pgs: 97 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.6 MiB/s wr, 474 op/s 2024-09-10T03:32:11.019 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:11.019 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:11.019 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-10T03:32:11.019 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:11.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 131217 11.32 MB/sec execute 200 sec latency 74.483 ms 2024-09-10T03:32:11.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:10 smithi164 ceph-mon[49318]: pgmap v23: 97 pgs: 97 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.6 MiB/s wr, 474 op/s 2024-09-10T03:32:11.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:11.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:11.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-10T03:32:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:12.046 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:11 smithi130 ceph-mon[63591]: Reconfiguring osd.3 (monmap changed)... 2024-09-10T03:32:12.047 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:11 smithi130 ceph-mon[63591]: Reconfiguring daemon osd.3 on smithi130 2024-09-10T03:32:12.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 131521 11.30 MB/sec execute 201 sec latency 63.541 ms 2024-09-10T03:32:12.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:11 smithi164 ceph-mon[49318]: Reconfiguring osd.3 (monmap changed)... 2024-09-10T03:32:12.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:11 smithi164 ceph-mon[49318]: Reconfiguring daemon osd.3 on smithi130 2024-09-10T03:32:13.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 131968 11.29 MB/sec execute 202 sec latency 32.087 ms 2024-09-10T03:32:13.067 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:12 smithi130 ceph-mon[63591]: pgmap v24: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 688 op/s 2024-09-10T03:32:13.067 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:12 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:13.068 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:12 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:13.068 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:12 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi130.ukcxnb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-10T03:32:13.068 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:12 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:13.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:12 smithi164 ceph-mon[49318]: pgmap v24: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 688 op/s 2024-09-10T03:32:13.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:12 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:13.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:12 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:13.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:12 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi130.ukcxnb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-10T03:32:13.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:12 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:14.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 132325 11.28 MB/sec execute 203 sec latency 66.806 ms 2024-09-10T03:32:14.086 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:14 smithi130 ceph-mon[63591]: Reconfiguring mds.foofs.smithi130.ukcxnb (monmap changed)... 2024-09-10T03:32:14.087 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:14 smithi130 ceph-mon[63591]: Reconfiguring daemon mds.foofs.smithi130.ukcxnb on smithi130 2024-09-10T03:32:14.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:14 smithi164 ceph-mon[49318]: Reconfiguring mds.foofs.smithi130.ukcxnb (monmap changed)... 2024-09-10T03:32:14.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:14 smithi164 ceph-mon[49318]: Reconfiguring daemon mds.foofs.smithi130.ukcxnb on smithi130 2024-09-10T03:32:15.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 132732 11.28 MB/sec execute 204 sec latency 100.078 ms 2024-09-10T03:32:15.100 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:15 smithi130 ceph-mon[63591]: pgmap v25: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 459 op/s 2024-09-10T03:32:15.358 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:15 smithi164 ceph-mon[49318]: pgmap v25: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 459 op/s 2024-09-10T03:32:15.700 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:32:16.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 133105 11.24 MB/sec execute 205 sec latency 44.365 ms 2024-09-10T03:32:16.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:16 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:32:16.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:16 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:16.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:16 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:16.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:16 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi164", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-10T03:32:16.120 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:16 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:16.124 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:16 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:32:16.124 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:16 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:16.124 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:16 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:16.124 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:16 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi164", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-10T03:32:16.124 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:16 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:16.800 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:32:16.801 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (3m) 37s ago 12m 20.0M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:32:16.801 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (11m) 37s ago 11m 6400k - 17.2.0 e1d6a67b021e 138ed3222ba5 2024-09-10T03:32:16.801 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (11m) 23s ago 11m 6405k - 17.2.0 e1d6a67b021e 3623793da55d 2024-09-10T03:32:16.801 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (2m) 37s ago 11m 56.2M - dad864ee21e9 72ba01ac1871 2024-09-10T03:32:16.801 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (5m) 37s ago 5m 36.8M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:32:16.801 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (5m) 23s ago 5m 15.7M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:32:16.801 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (2m) 37s ago 13m 488M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:32:16.801 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283,8765 running (76s) 23s ago 11m 452M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e4034e8e2a6d 2024-09-10T03:32:16.801 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (57s) 37s ago 13m 34.6M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 22e5cf56527a 2024-09-10T03:32:16.801 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (29s) 23s ago 11m 24.8M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 7d80bd0d15b9 2024-09-10T03:32:16.801 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (5m) 37s ago 5m 253M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:32:16.802 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (5m) 23s ago 5m 87.4M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:32:16.802 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (2m) 37s ago 11m 10.0M - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:32:16.802 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (2m) 23s ago 11m 10.0M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:32:16.802 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (10m) 37s ago 10m 457M 1936M 17.2.0 e1d6a67b021e 1d66043abecd 2024-09-10T03:32:16.802 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (9m) 37s ago 9m 383M 1936M 17.2.0 e1d6a67b021e ca1ec04aecfa 2024-09-10T03:32:16.802 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (9m) 37s ago 9m 278M 1936M 17.2.0 e1d6a67b021e 5883f5d5e969 2024-09-10T03:32:16.802 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (9m) 37s ago 9m 415M 1936M 17.2.0 e1d6a67b021e b32084e1a404 2024-09-10T03:32:16.802 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (8m) 23s ago 8m 444M 2704M 17.2.0 e1d6a67b021e 3b5c9f00f64d 2024-09-10T03:32:16.802 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (8m) 23s ago 8m 467M 2704M 17.2.0 e1d6a67b021e f16357ac331e 2024-09-10T03:32:16.802 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (7m) 23s ago 7m 407M 2704M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:32:16.802 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (7m) 23s ago 7m 571M 2704M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:32:16.802 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (82s) 37s ago 10m 46.1M - 2.51.0 1d3b7f56885b f59b02fac5b3 2024-09-10T03:32:17.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 133537 11.24 MB/sec execute 206 sec latency 54.098 ms 2024-09-10T03:32:17.131 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:17 smithi130 ceph-mon[63591]: Reconfiguring crash.smithi164 (monmap changed)... 2024-09-10T03:32:17.132 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:17 smithi130 ceph-mon[63591]: Reconfiguring daemon crash.smithi164 on smithi164 2024-09-10T03:32:17.132 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:17 smithi130 ceph-mon[63591]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:32:17.132 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:17 smithi130 ceph-mon[63591]: pgmap v26: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 459 op/s 2024-09-10T03:32:17.132 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:17 smithi130 ceph-mon[63591]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:32:17.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:17 smithi164 ceph-mon[49318]: Reconfiguring crash.smithi164 (monmap changed)... 2024-09-10T03:32:17.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:17 smithi164 ceph-mon[49318]: Reconfiguring daemon crash.smithi164 on smithi164 2024-09-10T03:32:17.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:17 smithi164 ceph-mon[49318]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:32:17.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:17 smithi164 ceph-mon[49318]: pgmap v26: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 459 op/s 2024-09-10T03:32:17.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:17 smithi164 ceph-mon[49318]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:32:17.492 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:32:17.492 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:32:17.492 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:32:17.492 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:32:17.492 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:32:17.492 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:32:17.492 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:32:17.492 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:32:17.492 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-10T03:32:17.492 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:32:17.492 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:32:17.493 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:32:17.493 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:32:17.493 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:32:17.493 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-10T03:32:17.493 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 4 2024-09-10T03:32:17.493 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:32:17.493 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:32:18.013 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:18 smithi164 ceph-mon[49318]: from='client.44107 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:32:18.013 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:18 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/1302091895' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:32:18.013 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:18 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:18.013 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:18 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:18.013 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:18 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi164.bkfwap", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-10T03:32:18.013 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:18 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mgr services"} : dispatch 2024-09-10T03:32:18.013 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:18 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:18.056 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:18 smithi130 ceph-mon[63591]: from='client.44107 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:32:18.057 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:18 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/1302091895' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:32:18.057 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:18 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:18.057 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:18 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:18.057 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:18 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi164.bkfwap", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-10T03:32:18.057 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:18 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mgr services"} : dispatch 2024-09-10T03:32:18.057 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:18 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:18.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 134139 11.30 MB/sec execute 207 sec latency 64.665 ms 2024-09-10T03:32:18.116 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:32:18.117 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:32:18.117 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:32:18.117 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:32:18.117 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-09-10T03:32:18.117 INFO:teuthology.orchestra.run.smithi130.stdout: "mon", 2024-09-10T03:32:18.117 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-09-10T03:32:18.117 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-09-10T03:32:18.117 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "4/23 daemons upgraded", 2024-09-10T03:32:18.117 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading mon daemons", 2024-09-10T03:32:18.117 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:32:18.117 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:32:18.841 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-09-10T03:32:19.061 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:19 smithi130 ceph-mon[63591]: pgmap v27: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.8 MiB/s wr, 692 op/s 2024-09-10T03:32:19.061 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:19 smithi130 ceph-mon[63591]: Reconfiguring mgr.smithi164.bkfwap (monmap changed)... 2024-09-10T03:32:19.061 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:19 smithi130 ceph-mon[63591]: Reconfiguring daemon mgr.smithi164.bkfwap on smithi164 2024-09-10T03:32:19.061 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:19 smithi130 ceph-mon[63591]: from='client.44115 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:32:19.061 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:19 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/105830388' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:32:19.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 134804 11.35 MB/sec execute 208 sec latency 25.212 ms 2024-09-10T03:32:19.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:19 smithi164 ceph-mon[49318]: pgmap v27: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.8 MiB/s wr, 692 op/s 2024-09-10T03:32:19.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:19 smithi164 ceph-mon[49318]: Reconfiguring mgr.smithi164.bkfwap (monmap changed)... 2024-09-10T03:32:19.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:19 smithi164 ceph-mon[49318]: Reconfiguring daemon mgr.smithi164.bkfwap on smithi164 2024-09-10T03:32:19.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:19 smithi164 ceph-mon[49318]: from='client.44115 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:32:19.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:19 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/105830388' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:32:20.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 135136 11.33 MB/sec execute 209 sec latency 60.700 ms 2024-09-10T03:32:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:20 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:20 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:20 smithi130 ceph-mon[63591]: Reconfiguring mon.smithi164 (monmap changed)... 2024-09-10T03:32:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:20 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-10T03:32:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:20 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-10T03:32:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:20 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:20 smithi130 ceph-mon[63591]: Reconfiguring daemon mon.smithi164 on smithi164 2024-09-10T03:32:20.341 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:20 smithi130 ceph-mon[63591]: pgmap v28: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 448 op/s 2024-09-10T03:32:20.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:20 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:20.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:20 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:20.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:20 smithi164 ceph-mon[49318]: Reconfiguring mon.smithi164 (monmap changed)... 2024-09-10T03:32:20.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:20 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-10T03:32:20.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:20 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-10T03:32:20.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:20 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:20.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:20 smithi164 ceph-mon[49318]: Reconfiguring daemon mon.smithi164 on smithi164 2024-09-10T03:32:20.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:20 smithi164 ceph-mon[49318]: pgmap v28: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 448 op/s 2024-09-10T03:32:21.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 135446 11.30 MB/sec execute 210 sec latency 83.146 ms 2024-09-10T03:32:21.893 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:21 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:21.893 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:21 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:21.893 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:21 smithi130 ceph-mon[63591]: Reconfiguring osd.4 (monmap changed)... 2024-09-10T03:32:21.893 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:21 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-10T03:32:21.893 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:21 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:21.893 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:21 smithi130 ceph-mon[63591]: Reconfiguring daemon osd.4 on smithi164 2024-09-10T03:32:22.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 135799 11.30 MB/sec execute 211 sec latency 51.667 ms 2024-09-10T03:32:22.189 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:21 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:22.189 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:21 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:22.189 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:21 smithi164 ceph-mon[49318]: Reconfiguring osd.4 (monmap changed)... 2024-09-10T03:32:22.189 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:21 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-10T03:32:22.189 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:21 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:22.189 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:21 smithi164 ceph-mon[49318]: Reconfiguring daemon osd.4 on smithi164 2024-09-10T03:32:23.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 136469 11.31 MB/sec execute 212 sec latency 32.602 ms 2024-09-10T03:32:23.399 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:23 smithi130 ceph-mon[63591]: pgmap v29: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.7 MiB/s wr, 733 op/s 2024-09-10T03:32:23.399 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:23 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:23.399 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:23 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:23.399 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:23 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-10T03:32:23.399 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:23 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:23.649 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:23 smithi164 ceph-mon[49318]: pgmap v29: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.7 MiB/s wr, 733 op/s 2024-09-10T03:32:23.649 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:23 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:23.649 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:23 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:23.649 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:23 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-10T03:32:23.649 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:23 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:24.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 137217 11.34 MB/sec execute 213 sec latency 20.165 ms 2024-09-10T03:32:24.403 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:24 smithi130 ceph-mon[63591]: Reconfiguring osd.5 (monmap changed)... 2024-09-10T03:32:24.403 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:24 smithi130 ceph-mon[63591]: Reconfiguring daemon osd.5 on smithi164 2024-09-10T03:32:24.403 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:24 smithi130 ceph-mon[63591]: pgmap v30: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.4 MiB/s wr, 519 op/s 2024-09-10T03:32:24.403 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:24 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:24.403 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:24 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:24.403 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:24 smithi130 ceph-mon[63591]: Reconfiguring osd.6 (monmap changed)... 2024-09-10T03:32:24.403 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:24 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-10T03:32:24.403 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:24 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:24.403 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:24 smithi130 ceph-mon[63591]: Reconfiguring daemon osd.6 on smithi164 2024-09-10T03:32:24.636 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:24 smithi164 ceph-mon[49318]: Reconfiguring osd.5 (monmap changed)... 2024-09-10T03:32:24.636 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:24 smithi164 ceph-mon[49318]: Reconfiguring daemon osd.5 on smithi164 2024-09-10T03:32:24.636 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:24 smithi164 ceph-mon[49318]: pgmap v30: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.4 MiB/s wr, 519 op/s 2024-09-10T03:32:24.636 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:24 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:24.636 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:24 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:24.636 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:24 smithi164 ceph-mon[49318]: Reconfiguring osd.6 (monmap changed)... 2024-09-10T03:32:24.636 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:24 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-10T03:32:24.636 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:24 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:24.636 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:24 smithi164 ceph-mon[49318]: Reconfiguring daemon osd.6 on smithi164 2024-09-10T03:32:25.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 138059 11.42 MB/sec execute 214 sec latency 35.788 ms 2024-09-10T03:32:26.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 138707 11.46 MB/sec execute 215 sec latency 39.184 ms 2024-09-10T03:32:26.671 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:26 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:26.671 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:26 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:26.671 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:26 smithi130 ceph-mon[63591]: Reconfiguring osd.7 (monmap changed)... 2024-09-10T03:32:26.671 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:26 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-10T03:32:26.671 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:26 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:26.671 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:26 smithi130 ceph-mon[63591]: Reconfiguring daemon osd.7 on smithi164 2024-09-10T03:32:26.671 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:26 smithi130 ceph-mon[63591]: pgmap v31: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 736 op/s 2024-09-10T03:32:26.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:26 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:26.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:26 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:26.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:26 smithi164 ceph-mon[49318]: Reconfiguring osd.7 (monmap changed)... 2024-09-10T03:32:26.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:26 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-10T03:32:26.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:26 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:26.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:26 smithi164 ceph-mon[49318]: Reconfiguring daemon osd.7 on smithi164 2024-09-10T03:32:26.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:26 smithi164 ceph-mon[49318]: pgmap v31: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 736 op/s 2024-09-10T03:32:27.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 139349 11.47 MB/sec execute 216 sec latency 27.225 ms 2024-09-10T03:32:28.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 139875 11.49 MB/sec execute 217 sec latency 36.271 ms 2024-09-10T03:32:28.437 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:28 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:28.437 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:28 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:28.437 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:28 smithi130 ceph-mon[63591]: Reconfiguring mds.foofs.smithi164.awiezo (monmap changed)... 2024-09-10T03:32:28.437 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:28 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi164.awiezo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-10T03:32:28.437 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:28 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:28.437 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:28 smithi130 ceph-mon[63591]: Reconfiguring daemon mds.foofs.smithi164.awiezo on smithi164 2024-09-10T03:32:28.678 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:28 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:28.678 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:28 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:28.678 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:28 smithi164 ceph-mon[49318]: Reconfiguring mds.foofs.smithi164.awiezo (monmap changed)... 2024-09-10T03:32:28.679 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:28 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi164.awiezo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-10T03:32:28.679 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:28 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:28.679 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:28 smithi164 ceph-mon[49318]: Reconfiguring daemon mds.foofs.smithi164.awiezo on smithi164 2024-09-10T03:32:29.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 140476 11.47 MB/sec execute 218 sec latency 42.502 ms 2024-09-10T03:32:29.445 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:29 smithi130 ceph-mon[63591]: pgmap v32: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 891 op/s 2024-09-10T03:32:29.445 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:29 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:29.445 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:29 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:29.445 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:29 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:32:29.445 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:29 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:32:29.445 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:29 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:32:29.445 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:29 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:29.445 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:29 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi130"} : dispatch 2024-09-10T03:32:29.446 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:29 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi130"}]': finished 2024-09-10T03:32:29.446 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:29 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi164"} : dispatch 2024-09-10T03:32:29.446 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:29 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi164"}]': finished 2024-09-10T03:32:29.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:29 smithi164 ceph-mon[49318]: pgmap v32: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 891 op/s 2024-09-10T03:32:29.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:29 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:29.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:29 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:29.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:29 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:32:29.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:29 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:32:29.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:29 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:32:29.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:29 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:29.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:29 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi130"} : dispatch 2024-09-10T03:32:29.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:29 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi130"}]': finished 2024-09-10T03:32:29.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:29 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi164"} : dispatch 2024-09-10T03:32:29.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:29 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi164"}]': finished 2024-09-10T03:32:30.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 141116 11.54 MB/sec execute 219 sec latency 27.225 ms 2024-09-10T03:32:30.460 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:30 smithi130 ceph-mon[63591]: Upgrade: Setting container_image for all mon 2024-09-10T03:32:30.460 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:32:30.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:30 smithi164 ceph-mon[49318]: Upgrade: Setting container_image for all mon 2024-09-10T03:32:30.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:32:31.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 141720 11.57 MB/sec execute 220 sec latency 40.764 ms 2024-09-10T03:32:31.467 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:31 smithi130 ceph-mon[63591]: pgmap v33: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.1 MiB/s wr, 656 op/s 2024-09-10T03:32:31.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:31 smithi164 ceph-mon[49318]: pgmap v33: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.1 MiB/s wr, 656 op/s 2024-09-10T03:32:32.063 INFO:teuthology.orchestra.run.smithi130.stdout: 5 142053 11.56 MB/sec execute 221 sec latency 39.962 ms 2024-09-10T03:32:33.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 142386 11.54 MB/sec execute 222 sec latency 43.438 ms 2024-09-10T03:32:33.493 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:33 smithi130 ceph-mon[63591]: pgmap v34: 97 pgs: 97 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.00k op/s 2024-09-10T03:32:33.493 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:33 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:33.493 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:33 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi130", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-10T03:32:33.493 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:33 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:33.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:33 smithi164 ceph-mon[49318]: pgmap v34: 97 pgs: 97 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.00k op/s 2024-09-10T03:32:33.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:33 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:33.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:33 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi130", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-10T03:32:33.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:33 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:34.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 142840 11.54 MB/sec execute 223 sec latency 37.082 ms 2024-09-10T03:32:34.503 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:34 smithi130 ceph-mon[63591]: Upgrade: Updating crash.smithi130 (1/2) 2024-09-10T03:32:34.503 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:34 smithi130 ceph-mon[63591]: Deploying daemon crash.smithi130 on smithi130 2024-09-10T03:32:34.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:34 smithi164 ceph-mon[49318]: Upgrade: Updating crash.smithi130 (1/2) 2024-09-10T03:32:34.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:34 smithi164 ceph-mon[49318]: Deploying daemon crash.smithi130 on smithi130 2024-09-10T03:32:35.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 143409 11.56 MB/sec execute 224 sec latency 37.209 ms 2024-09-10T03:32:35.525 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:35 smithi130 ceph-mon[63591]: pgmap v35: 97 pgs: 97 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 716 op/s 2024-09-10T03:32:35.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:35 smithi164 ceph-mon[49318]: pgmap v35: 97 pgs: 97 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 716 op/s 2024-09-10T03:32:36.063 INFO:teuthology.orchestra.run.smithi130.stdout: 5 143960 11.54 MB/sec execute 225 sec latency 42.109 ms 2024-09-10T03:32:36.543 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:36 smithi130 ceph-mon[63591]: pgmap v36: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 8.9 MiB/s wr, 855 op/s 2024-09-10T03:32:36.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:36 smithi164 ceph-mon[49318]: pgmap v36: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 8.9 MiB/s wr, 855 op/s 2024-09-10T03:32:37.063 INFO:teuthology.orchestra.run.smithi130.stdout: 5 144434 11.56 MB/sec execute 226 sec latency 31.483 ms 2024-09-10T03:32:38.062 INFO:teuthology.orchestra.run.smithi130.stdout: 5 144986 11.59 MB/sec execute 227 sec latency 52.181 ms 2024-09-10T03:32:38.926 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:38 smithi130 ceph-mon[63591]: pgmap v37: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 7.8 MiB/s wr, 776 op/s 2024-09-10T03:32:39.063 INFO:teuthology.orchestra.run.smithi130.stdout: 5 145444 11.60 MB/sec execute 228 sec latency 39.496 ms 2024-09-10T03:32:39.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:38 smithi164 ceph-mon[49318]: pgmap v37: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 7.8 MiB/s wr, 776 op/s 2024-09-10T03:32:40.063 INFO:teuthology.orchestra.run.smithi130.stdout: 5 145705 11.59 MB/sec execute 229 sec latency 48.616 ms 2024-09-10T03:32:40.197 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:39 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:40.197 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:39 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:40.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:39 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:40.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:39 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:41.063 INFO:teuthology.orchestra.run.smithi130.stdout: 5 145939 11.56 MB/sec execute 230 sec latency 47.689 ms 2024-09-10T03:32:41.226 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:40 smithi130 ceph-mon[63591]: pgmap v38: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.1 MiB/s wr, 622 op/s 2024-09-10T03:32:41.227 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:40 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:41.227 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:40 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi164", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-10T03:32:41.227 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:40 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:41.334 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:40 smithi164 ceph-mon[49318]: pgmap v38: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.1 MiB/s wr, 622 op/s 2024-09-10T03:32:41.334 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:40 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:41.334 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:40 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi164", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-10T03:32:41.335 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:40 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:32:42.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:41 smithi130 ceph-mon[63591]: Upgrade: Updating crash.smithi164 (2/2) 2024-09-10T03:32:42.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:41 smithi130 ceph-mon[63591]: Deploying daemon crash.smithi164 on smithi164 2024-09-10T03:32:42.063 INFO:teuthology.orchestra.run.smithi130.stdout: 5 146206 11.54 MB/sec execute 231 sec latency 96.848 ms 2024-09-10T03:32:42.125 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:42 smithi164 ceph-mon[49318]: Upgrade: Updating crash.smithi164 (2/2) 2024-09-10T03:32:42.125 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:42 smithi164 ceph-mon[49318]: Deploying daemon crash.smithi164 on smithi164 2024-09-10T03:32:43.063 INFO:teuthology.orchestra.run.smithi130.stdout: 5 146459 11.50 MB/sec execute 232 sec latency 66.489 ms 2024-09-10T03:32:43.075 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:43 smithi130 ceph-mon[63591]: pgmap v39: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 884 op/s 2024-09-10T03:32:43.401 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:43 smithi164 ceph-mon[49318]: pgmap v39: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 884 op/s 2024-09-10T03:32:44.063 INFO:teuthology.orchestra.run.smithi130.stdout: 5 146650 11.48 MB/sec execute 233 sec latency 71.992 ms 2024-09-10T03:32:45.063 INFO:teuthology.orchestra.run.smithi130.stdout: 5 146920 11.47 MB/sec execute 234 sec latency 48.538 ms 2024-09-10T03:32:45.136 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:45 smithi130 ceph-mon[63591]: pgmap v40: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.6 MiB/s wr, 539 op/s 2024-09-10T03:32:45.136 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:45 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:45.136 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:45 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:45.136 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:45 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:32:45.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:45 smithi164 ceph-mon[49318]: pgmap v40: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.6 MiB/s wr, 539 op/s 2024-09-10T03:32:45.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:45 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:45.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:45 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:45.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:45 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:32:46.063 INFO:teuthology.orchestra.run.smithi130.stdout: 5 147690 11.47 MB/sec execute 235 sec latency 52.758 ms 2024-09-10T03:32:46.140 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:46 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:32:46.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:46 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:32:47.063 INFO:teuthology.orchestra.run.smithi130.stdout: 5 148372 11.54 MB/sec execute 236 sec latency 26.525 ms 2024-09-10T03:32:47.143 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:47 smithi130 ceph-mon[63591]: pgmap v41: 97 pgs: 97 active+clean; 327 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 718 op/s 2024-09-10T03:32:47.285 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:47 smithi164 ceph-mon[49318]: pgmap v41: 97 pgs: 97 active+clean; 327 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 718 op/s 2024-09-10T03:32:48.063 INFO:teuthology.orchestra.run.smithi130.stdout: 5 148980 11.57 MB/sec execute 237 sec latency 41.327 ms 2024-09-10T03:32:49.064 INFO:teuthology.orchestra.run.smithi130.stdout: 5 149296 11.56 MB/sec execute 238 sec latency 34.092 ms 2024-09-10T03:32:49.167 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:49 smithi130 ceph-mon[63591]: pgmap v42: 97 pgs: 97 active+clean; 327 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.5 MiB/s wr, 586 op/s 2024-09-10T03:32:49.168 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:49 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:49.168 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:49 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:49.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:49 smithi164 ceph-mon[49318]: pgmap v42: 97 pgs: 97 active+clean; 327 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.5 MiB/s wr, 586 op/s 2024-09-10T03:32:49.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:49 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:49.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:49 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:49.473 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:32:50.064 INFO:teuthology.orchestra.run.smithi130.stdout: 5 149664 11.54 MB/sec execute 239 sec latency 54.261 ms 2024-09-10T03:32:50.451 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:50 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:50.451 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:50 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:50.451 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:50 smithi130 ceph-mon[63591]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:32:50.451 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:50 smithi130 ceph-mon[63591]: pgmap v43: 97 pgs: 97 active+clean; 327 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.5 MiB/s wr, 448 op/s 2024-09-10T03:32:50.451 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:50 smithi130 ceph-mon[63591]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:32:50.664 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:32:50.665 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (3m) 71s ago 12m 20.0M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:32:50.665 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 starting - - - - 2024-09-10T03:32:50.665 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (6s) 1s ago 11m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 243c094648b8 2024-09-10T03:32:50.665 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (3m) 71s ago 12m 56.2M - dad864ee21e9 72ba01ac1871 2024-09-10T03:32:50.665 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (6m) 71s ago 6m 36.8M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:32:50.665 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (6m) 1s ago 6m 15.9M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:32:50.665 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (2m) 71s ago 13m 488M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:32:50.665 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283,8765 running (109s) 1s ago 11m 453M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e4034e8e2a6d 2024-09-10T03:32:50.665 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (91s) 71s ago 13m 34.6M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 22e5cf56527a 2024-09-10T03:32:50.665 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (63s) 1s ago 11m 36.8M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 7d80bd0d15b9 2024-09-10T03:32:50.665 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (6m) 71s ago 6m 253M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:32:50.665 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (6m) 1s ago 6m 87.5M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:32:50.665 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (3m) 71s ago 12m 10.0M - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:32:50.666 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (2m) 1s ago 11m 10.6M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:32:50.666 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (11m) 71s ago 11m 457M 1936M 17.2.0 e1d6a67b021e 1d66043abecd 2024-09-10T03:32:50.666 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (10m) 71s ago 10m 383M 1936M 17.2.0 e1d6a67b021e ca1ec04aecfa 2024-09-10T03:32:50.666 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (10m) 71s ago 10m 278M 1936M 17.2.0 e1d6a67b021e 5883f5d5e969 2024-09-10T03:32:50.666 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (9m) 71s ago 9m 415M 1936M 17.2.0 e1d6a67b021e b32084e1a404 2024-09-10T03:32:50.666 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (9m) 1s ago 9m 504M 2704M 17.2.0 e1d6a67b021e 3b5c9f00f64d 2024-09-10T03:32:50.666 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (8m) 1s ago 8m 559M 2704M 17.2.0 e1d6a67b021e f16357ac331e 2024-09-10T03:32:50.666 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (8m) 1s ago 8m 490M 2704M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:32:50.666 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (8m) 1s ago 8m 396M 2704M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:32:50.666 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (116s) 71s ago 11m 46.1M - 2.51.0 1d3b7f56885b f59b02fac5b3 2024-09-10T03:32:50.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:50 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:50.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:50 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:50.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:50 smithi164 ceph-mon[49318]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:32:50.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:50 smithi164 ceph-mon[49318]: pgmap v43: 97 pgs: 97 active+clean; 327 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.5 MiB/s wr, 448 op/s 2024-09-10T03:32:50.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:50 smithi164 ceph-mon[49318]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:32:51.064 INFO:teuthology.orchestra.run.smithi130.stdout: 5 150005 11.52 MB/sec execute 240 sec latency 51.364 ms 2024-09-10T03:32:51.345 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:32:51.345 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:32:51.345 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:32:51.345 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:32:51.345 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:32:51.346 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:32:51.346 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:32:51.346 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:32:51.346 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-10T03:32:51.346 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:32:51.346 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:32:51.346 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:32:51.346 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:32:51.346 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:32:51.346 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-10T03:32:51.346 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 4 2024-09-10T03:32:51.346 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:32:51.346 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:32:51.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:51 smithi164 ceph-mon[49318]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:32:51.474 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:51 smithi130 ceph-mon[63591]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:32:51.997 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:32:51.998 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:32:51.998 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:32:51.998 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:32:51.998 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-09-10T03:32:51.998 INFO:teuthology.orchestra.run.smithi130.stdout: "mon", 2024-09-10T03:32:51.998 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-09-10T03:32:51.998 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-09-10T03:32:51.998 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "5/23 daemons upgraded", 2024-09-10T03:32:51.998 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading crash daemons", 2024-09-10T03:32:51.998 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:32:51.998 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:32:52.064 INFO:teuthology.orchestra.run.smithi130.stdout: 5 150271 11.49 MB/sec execute 241 sec latency 69.170 ms 2024-09-10T03:32:52.498 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:52 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/708160878' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:32:52.498 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:52 smithi130 ceph-mon[63591]: pgmap v44: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 752 op/s 2024-09-10T03:32:52.498 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:52 smithi130 ceph-mon[63591]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:32:52.654 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:52 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/708160878' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:32:52.654 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:52 smithi164 ceph-mon[49318]: pgmap v44: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 752 op/s 2024-09-10T03:32:52.654 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:52 smithi164 ceph-mon[49318]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:32:52.716 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-09-10T03:32:53.064 INFO:teuthology.orchestra.run.smithi130.stdout: 5 150895 11.51 MB/sec execute 242 sec latency 31.818 ms 2024-09-10T03:32:53.754 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:53 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:53.754 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:53 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:53.754 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:53 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/1742020162' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:32:53.754 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:53 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:53.754 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:53 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:53.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:53 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:53.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:53 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:53.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:53 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/1742020162' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:32:53.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:53 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:53.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:53 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:54.064 INFO:teuthology.orchestra.run.smithi130.stdout: 5 151496 11.52 MB/sec execute 243 sec latency 17.099 ms 2024-09-10T03:32:55.064 INFO:teuthology.orchestra.run.smithi130.stdout: 5 152040 11.56 MB/sec execute 244 sec latency 58.096 ms 2024-09-10T03:32:55.261 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:55 smithi130 ceph-mon[63591]: pgmap v45: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.8 MiB/s wr, 490 op/s 2024-09-10T03:32:55.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:55 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:55.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:55 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:55.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:55 smithi164 ceph-mon[49318]: pgmap v45: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.8 MiB/s wr, 490 op/s 2024-09-10T03:32:55.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:55 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:55.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:55 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:32:56.064 INFO:teuthology.orchestra.run.smithi130.stdout: 5 152527 11.58 MB/sec execute 245 sec latency 36.779 ms 2024-09-10T03:32:57.064 INFO:teuthology.orchestra.run.smithi130.stdout: 5 152852 11.57 MB/sec execute 246 sec latency 30.152 ms 2024-09-10T03:32:57.284 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:57 smithi130 ceph-mon[63591]: pgmap v46: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 772 op/s 2024-09-10T03:32:57.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:57 smithi164 ceph-mon[49318]: pgmap v46: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 772 op/s 2024-09-10T03:32:58.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 153443 11.57 MB/sec execute 247 sec latency 27.442 ms 2024-09-10T03:32:59.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 153950 11.57 MB/sec execute 248 sec latency 41.712 ms 2024-09-10T03:32:59.294 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:32:59 smithi130 ceph-mon[63591]: pgmap v47: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.5 MiB/s wr, 602 op/s 2024-09-10T03:32:59.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:32:59 smithi164 ceph-mon[49318]: pgmap v47: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.5 MiB/s wr, 602 op/s 2024-09-10T03:33:00.064 INFO:teuthology.orchestra.run.smithi130.stdout: 5 154459 11.58 MB/sec execute 249 sec latency 43.414 ms 2024-09-10T03:33:00.316 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:33:00.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:33:01.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 154876 11.57 MB/sec execute 250 sec latency 57.293 ms 2024-09-10T03:33:01.328 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:01 smithi130 ceph-mon[63591]: pgmap v48: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.4 MiB/s wr, 595 op/s 2024-09-10T03:33:01.328 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:01.328 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:01.328 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:33:01.328 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:33:01.328 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:01.328 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:01.328 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:33:01.328 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:33:01.328 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:33:01.329 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:33:01.329 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:01.329 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi130"} : dispatch 2024-09-10T03:33:01.329 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi130"}]': finished 2024-09-10T03:33:01.329 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi164"} : dispatch 2024-09-10T03:33:01.329 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi164"}]': finished 2024-09-10T03:33:01.329 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-10T03:33:01.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:01 smithi164 ceph-mon[49318]: pgmap v48: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.4 MiB/s wr, 595 op/s 2024-09-10T03:33:01.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:33:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:33:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:33:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:33:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:33:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:33:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi130"} : dispatch 2024-09-10T03:33:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi130"}]': finished 2024-09-10T03:33:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi164"} : dispatch 2024-09-10T03:33:01.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi164"}]': finished 2024-09-10T03:33:01.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-10T03:33:02.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 155128 11.55 MB/sec execute 251 sec latency 53.488 ms 2024-09-10T03:33:02.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:02 smithi130 ceph-mon[63591]: Upgrade: Setting container_image for all crash 2024-09-10T03:33:02.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:02 smithi130 ceph-mon[63591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-10T03:33:02.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:02 smithi130 ceph-mon[63591]: Upgrade: osd.0 is safe to restart 2024-09-10T03:33:02.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:02 smithi164 ceph-mon[49318]: Upgrade: Setting container_image for all crash 2024-09-10T03:33:02.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:02 smithi164 ceph-mon[49318]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-10T03:33:02.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:02 smithi164 ceph-mon[49318]: Upgrade: osd.0 is safe to restart 2024-09-10T03:33:03.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 155487 11.56 MB/sec execute 252 sec latency 83.000 ms 2024-09-10T03:33:03.355 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:03 smithi130 ceph-mon[63591]: pgmap v49: 97 pgs: 97 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 891 op/s 2024-09-10T03:33:03.355 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:03 smithi130 ceph-mon[63591]: mgrmap e43: smithi130.axcmel(active, since 92s), standbys: smithi164.bkfwap 2024-09-10T03:33:03.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:03 smithi164 ceph-mon[49318]: pgmap v49: 97 pgs: 97 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 891 op/s 2024-09-10T03:33:03.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:03 smithi164 ceph-mon[49318]: mgrmap e43: smithi130.axcmel(active, since 92s), standbys: smithi164.bkfwap 2024-09-10T03:33:04.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 155888 11.55 MB/sec execute 253 sec latency 65.136 ms 2024-09-10T03:33:04.627 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:04 smithi130 ceph-mon[63591]: Upgrade: Updating osd.0 2024-09-10T03:33:04.627 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:04 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:04.627 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:04 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-10T03:33:04.627 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:04 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:33:04.627 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:04 smithi130 ceph-mon[63591]: Deploying daemon osd.0 on smithi130 2024-09-10T03:33:04.627 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:04 smithi130 ceph-mon[63591]: pgmap v50: 97 pgs: 97 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.7 MiB/s wr, 587 op/s 2024-09-10T03:33:04.640 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:04 smithi164 ceph-mon[49318]: Upgrade: Updating osd.0 2024-09-10T03:33:04.640 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:04 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:04.640 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:04 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-10T03:33:04.640 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:04 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:33:04.640 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:04 smithi164 ceph-mon[49318]: Deploying daemon osd.0 on smithi130 2024-09-10T03:33:04.640 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:04 smithi164 ceph-mon[49318]: pgmap v50: 97 pgs: 97 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.7 MiB/s wr, 587 op/s 2024-09-10T03:33:05.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 156216 11.56 MB/sec execute 254 sec latency 37.289 ms 2024-09-10T03:33:06.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 156506 11.56 MB/sec execute 255 sec latency 54.116 ms 2024-09-10T03:33:06.975 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:06 smithi130 ceph-mon[63591]: pgmap v51: 97 pgs: 97 active+clean; 325 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 743 op/s 2024-09-10T03:33:07.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 156811 11.53 MB/sec execute 256 sec latency 47.734 ms 2024-09-10T03:33:07.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:06 smithi164 ceph-mon[49318]: pgmap v51: 97 pgs: 97 active+clean; 325 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 743 op/s 2024-09-10T03:33:08.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 157101 11.50 MB/sec execute 257 sec latency 57.266 ms 2024-09-10T03:33:08.994 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:08 smithi130 ceph-mon[63591]: pgmap v52: 97 pgs: 97 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.3 MiB/s wr, 511 op/s 2024-09-10T03:33:08.994 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:08 smithi130 systemd[1]: Stopping Ceph osd.0 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:33:09.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 157304 11.48 MB/sec execute 258 sec latency 48.183 ms 2024-09-10T03:33:09.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:08 smithi164 ceph-mon[49318]: pgmap v52: 97 pgs: 97 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.3 MiB/s wr, 511 op/s 2024-09-10T03:33:09.255 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:09 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0[34066]: 2024-09-10T03:33:09.113+0000 7fe30c909700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:33:09.255 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:09 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0[34066]: 2024-09-10T03:33:09.113+0000 7fe30c909700 -1 osd.0 73 *** Got signal Terminated *** 2024-09-10T03:33:09.255 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:09 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0[34066]: 2024-09-10T03:33:09.113+0000 7fe30c909700 -1 osd.0 73 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-10T03:33:10.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:09 smithi130 ceph-mon[63591]: osd.0 marked itself down and dead 2024-09-10T03:33:10.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 157629 11.48 MB/sec execute 259 sec latency 252.809 ms 2024-09-10T03:33:10.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:09 smithi164 ceph-mon[49318]: osd.0 marked itself down and dead 2024-09-10T03:33:10.728 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:10 smithi130 podman[67881]: 2024-09-10 03:33:10.484898404 +0000 UTC m=+1.546802671 container died 1d66043abecdac4fc5373c5ecdfd84c6db291538e41b556e112a20eb48f84d15 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0, maintainer=Guillaume Abrioux , version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.expose-services=, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=HEAD, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, name=centos-stream, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc.) 2024-09-10T03:33:10.978 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:10 smithi130 ceph-mon[63591]: pgmap v53: 97 pgs: 97 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.1 MiB/s wr, 501 op/s 2024-09-10T03:33:10.979 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:10 smithi130 ceph-mon[63591]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-10T03:33:10.979 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:10 smithi130 ceph-mon[63591]: osdmap e74: 8 total, 7 up, 8 in 2024-09-10T03:33:10.979 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:10 smithi130 podman[67881]: 2024-09-10 03:33:10.851805661 +0000 UTC m=+1.913709929 container cleanup 1d66043abecdac4fc5373c5ecdfd84c6db291538e41b556e112a20eb48f84d15 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, ceph=True, io.openshift.tags=base centos centos-stream, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, build-date=2022-05-03T08:36:31.336870, vcs-type=git, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, release=754, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD) 2024-09-10T03:33:10.979 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:10 smithi130 bash[67881]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0 2024-09-10T03:33:11.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 157728 11.44 MB/sec execute 260 sec latency 1018.234 ms 2024-09-10T03:33:11.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:10 smithi164 ceph-mon[49318]: pgmap v53: 97 pgs: 97 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.1 MiB/s wr, 501 op/s 2024-09-10T03:33:11.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:10 smithi164 ceph-mon[49318]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-10T03:33:11.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:10 smithi164 ceph-mon[49318]: osdmap e74: 8 total, 7 up, 8 in 2024-09-10T03:33:11.231 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:11 smithi130 podman[67896]: 2024-09-10 03:33:11.095324856 +0000 UTC m=+0.604747846 container remove 1d66043abecdac4fc5373c5ecdfd84c6db291538e41b556e112a20eb48f84d15 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0, architecture=x86_64, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, GIT_CLEAN=True, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, version=8, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-10T03:33:11.733 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:11 smithi130 podman[67982]: 2024-09-10 03:33:11.438630302 +0000 UTC m=+0.017103402 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:33:11.733 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:11 smithi130 podman[67982]: 2024-09-10 03:33:11.563187549 +0000 UTC m=+0.141660631 container create 2f0acf9ea6ebd34cdc06fedbac823e513c8eebd64c03572a2a6ef89a48e9e9d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-10T03:33:11.983 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:11 smithi130 ceph-mon[63591]: osdmap e75: 8 total, 7 up, 8 in 2024-09-10T03:33:11.984 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:11 smithi130 podman[67982]: 2024-09-10 03:33:11.910557809 +0000 UTC m=+0.489030895 container init 2f0acf9ea6ebd34cdc06fedbac823e513c8eebd64c03572a2a6ef89a48e9e9d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-10T03:33:11.984 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:11 smithi130 podman[67982]: 2024-09-10 03:33:11.9189628 +0000 UTC m=+0.497435886 container start 2f0acf9ea6ebd34cdc06fedbac823e513c8eebd64c03572a2a6ef89a48e9e9d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-10T03:33:12.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 158039 11.42 MB/sec execute 261 sec latency 73.573 ms 2024-09-10T03:33:12.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:11 smithi164 ceph-mon[49318]: osdmap e75: 8 total, 7 up, 8 in 2024-09-10T03:33:12.238 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:12 smithi130 podman[67982]: 2024-09-10 03:33:12.028839476 +0000 UTC m=+0.607312560 container attach 2f0acf9ea6ebd34cdc06fedbac823e513c8eebd64c03572a2a6ef89a48e9e9d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-10T03:33:12.238 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:12 smithi130 podman[67982]: 2024-09-10 03:33:12.226514143 +0000 UTC m=+0.804987235 container died 2f0acf9ea6ebd34cdc06fedbac823e513c8eebd64c03572a2a6ef89a48e9e9d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-deactivate, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-10T03:33:12.999 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:12 smithi130 ceph-mon[63591]: pgmap v56: 97 pgs: 36 peering, 61 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.0 MiB/s wr, 558 op/s 2024-09-10T03:33:12.999 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:12 smithi130 ceph-mon[63591]: Health check failed: Reduced data availability: 5 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-09-10T03:33:13.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 158561 11.42 MB/sec execute 262 sec latency 44.507 ms 2024-09-10T03:33:13.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:12 smithi164 ceph-mon[49318]: pgmap v56: 97 pgs: 36 peering, 61 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.0 MiB/s wr, 558 op/s 2024-09-10T03:33:13.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:12 smithi164 ceph-mon[49318]: Health check failed: Reduced data availability: 5 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-09-10T03:33:13.220 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:13 smithi130 podman[68005]: 2024-09-10 03:33:13.200055752 +0000 UTC m=+0.966949135 container remove 2f0acf9ea6ebd34cdc06fedbac823e513c8eebd64c03572a2a6ef89a48e9e9d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-10T03:33:13.251 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:13 smithi130 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-10T03:33:13.251 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:13 smithi130 systemd[1]: Stopped Ceph osd.0 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:33:13.251 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:13 smithi130 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.0.service: Consumed 2min 39.792s CPU time. 2024-09-10T03:33:13.754 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:13 smithi130 systemd[1]: Starting Ceph osd.0 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:33:14.005 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:13 smithi130 podman[68101]: 2024-09-10 03:33:13.776085754 +0000 UTC m=+0.022975202 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:33:14.005 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:13 smithi130 podman[68101]: 2024-09-10 03:33:13.918411305 +0000 UTC m=+0.165300745 container create 93acc4881f80fc2fded8e29f20ea7fb3755fa2dd4f019dac11605ca062bf7d41 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-activate, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, maintainer=Guillaume Abrioux ) 2024-09-10T03:33:14.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 159198 11.45 MB/sec execute 263 sec latency 30.661 ms 2024-09-10T03:33:14.259 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:14 smithi130 podman[68101]: 2024-09-10 03:33:14.254732689 +0000 UTC m=+0.501622143 container init 93acc4881f80fc2fded8e29f20ea7fb3755fa2dd4f019dac11605ca062bf7d41 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-activate, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True) 2024-09-10T03:33:14.510 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:14 smithi130 podman[68101]: 2024-09-10 03:33:14.26217907 +0000 UTC m=+0.509068514 container start 93acc4881f80fc2fded8e29f20ea7fb3755fa2dd4f019dac11605ca062bf7d41 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-activate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-10T03:33:14.510 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:14 smithi130 podman[68101]: 2024-09-10 03:33:14.430433454 +0000 UTC m=+0.677322901 container attach 93acc4881f80fc2fded8e29f20ea7fb3755fa2dd4f019dac11605ca062bf7d41 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0) 2024-09-10T03:33:14.510 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:14 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-activate[68115]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:33:14.510 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:14 smithi130 bash[68101]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:33:14.510 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:14 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-activate[68115]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:33:14.510 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:14 smithi130 bash[68101]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:33:15.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:14 smithi130 ceph-mon[63591]: pgmap v57: 97 pgs: 36 peering, 61 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.3 MiB/s wr, 325 op/s 2024-09-10T03:33:15.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 159938 11.51 MB/sec execute 264 sec latency 23.804 ms 2024-09-10T03:33:15.164 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:14 smithi164 ceph-mon[49318]: pgmap v57: 97 pgs: 36 peering, 61 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.3 MiB/s wr, 325 op/s 2024-09-10T03:33:16.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 160655 11.52 MB/sec execute 265 sec latency 38.559 ms 2024-09-10T03:33:16.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:16 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:16.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:16 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:33:16.270 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:16 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:16.270 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:16 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:33:16.270 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:16 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-activate[68115]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-10T03:33:16.270 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:16 smithi130 bash[68101]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-10T03:33:16.270 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:16 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-activate[68115]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-10T03:33:16.270 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:16 smithi130 bash[68101]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-10T03:33:16.521 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:16 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-activate[68115]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-10T03:33:16.521 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:16 smithi130 bash[68101]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-10T03:33:16.521 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:16 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-activate[68115]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-10T03:33:16.521 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:16 smithi130 bash[68101]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-10T03:33:16.521 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:16 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-activate[68115]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-10T03:33:16.521 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:16 smithi130 bash[68101]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-10T03:33:16.521 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:16 smithi130 bash[68101]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-10T03:33:16.522 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:16 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-activate[68115]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-10T03:33:16.522 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:16 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-activate[68115]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-10T03:33:16.522 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:16 smithi130 bash[68101]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-10T03:33:16.522 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:16 smithi130 podman[68101]: 2024-09-10 03:33:16.486482494 +0000 UTC m=+2.733371949 container died 93acc4881f80fc2fded8e29f20ea7fb3755fa2dd4f019dac11605ca062bf7d41 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.license=GPLv2) 2024-09-10T03:33:17.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 161284 11.55 MB/sec execute 266 sec latency 37.760 ms 2024-09-10T03:33:17.275 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:17 smithi130 ceph-mon[63591]: pgmap v58: 97 pgs: 4 active+undersized, 6 peering, 26 active+undersized+degraded, 61 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.6 MiB/s wr, 690 op/s; 220/1857 objects degraded (11.847%) 2024-09-10T03:33:17.276 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:17 smithi130 ceph-mon[63591]: Health check failed: Degraded data redundancy: 220/1857 objects degraded (11.847%), 26 pgs degraded (PG_DEGRADED) 2024-09-10T03:33:17.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:17 smithi164 ceph-mon[49318]: pgmap v58: 97 pgs: 4 active+undersized, 6 peering, 26 active+undersized+degraded, 61 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.6 MiB/s wr, 690 op/s; 220/1857 objects degraded (11.847%) 2024-09-10T03:33:17.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:17 smithi164 ceph-mon[49318]: Health check failed: Degraded data redundancy: 220/1857 objects degraded (11.847%), 26 pgs degraded (PG_DEGRADED) 2024-09-10T03:33:17.781 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:17 smithi130 podman[68267]: 2024-09-10 03:33:17.637403033 +0000 UTC m=+1.140478016 container remove 93acc4881f80fc2fded8e29f20ea7fb3755fa2dd4f019dac11605ca062bf7d41 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-activate, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-10T03:33:18.032 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:17 smithi130 podman[68324]: 2024-09-10 03:33:17.813537837 +0000 UTC m=+0.031184077 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:33:18.032 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:17 smithi130 podman[68324]: 2024-09-10 03:33:17.95525144 +0000 UTC m=+0.172897669 container create de3cbff0098b445a5127e8eac4942621716538b9faecfde12405069e5fdad44e (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-10T03:33:18.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 161993 11.56 MB/sec execute 267 sec latency 70.746 ms 2024-09-10T03:33:18.287 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:18 smithi130 ceph-mon[63591]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-09-10T03:33:18.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:18 smithi164 ceph-mon[49318]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-09-10T03:33:18.552 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:18 smithi130 podman[68324]: 2024-09-10 03:33:18.534184267 +0000 UTC m=+0.751830492 container init de3cbff0098b445a5127e8eac4942621716538b9faecfde12405069e5fdad44e (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-10T03:33:18.552 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:18 smithi130 podman[68324]: 2024-09-10 03:33:18.539291289 +0000 UTC m=+0.756937522 container start de3cbff0098b445a5127e8eac4942621716538b9faecfde12405069e5fdad44e (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-10T03:33:18.821 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:18 smithi130 ceph-osd[68343]: -- 172.21.15.130:0/2377683278 <== mon.0 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e36fe501e0 con 0x55e36fddc000 2024-09-10T03:33:18.821 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:18 smithi130 bash[68324]: de3cbff0098b445a5127e8eac4942621716538b9faecfde12405069e5fdad44e 2024-09-10T03:33:18.821 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:18 smithi130 systemd[1]: Started Ceph osd.0 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:33:19.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 162346 11.54 MB/sec execute 268 sec latency 54.732 ms 2024-09-10T03:33:19.326 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:19 smithi130 ceph-mon[63591]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.2 MiB/s wr, 715 op/s; 246/1908 objects degraded (12.893%) 2024-09-10T03:33:19.326 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:19 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:19.326 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:19 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:19.326 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:19 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:33:19.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:19 smithi164 ceph-mon[49318]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.2 MiB/s wr, 715 op/s; 246/1908 objects degraded (12.893%) 2024-09-10T03:33:19.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:19 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:19.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:19 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:19.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:19 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:33:20.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 162682 11.55 MB/sec execute 269 sec latency 49.780 ms 2024-09-10T03:33:20.850 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:20 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0[68339]: 2024-09-10T03:33:20.593+0000 7f77b3519640 -1 Falling back to public interface 2024-09-10T03:33:21.065 INFO:teuthology.orchestra.run.smithi130.stdout: 5 163021 11.53 MB/sec execute 270 sec latency 84.625 ms 2024-09-10T03:33:21.131 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:21 smithi130 ceph-mon[63591]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.8 MiB/s wr, 578 op/s; 246/1908 objects degraded (12.893%) 2024-09-10T03:33:21.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:21 smithi164 ceph-mon[49318]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.8 MiB/s wr, 578 op/s; 246/1908 objects degraded (12.893%) 2024-09-10T03:33:22.073 INFO:teuthology.orchestra.run.smithi130.stdout: 5 163369 11.56 MB/sec execute 271 sec latency 44.020 ms 2024-09-10T03:33:22.407 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:22 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 216/1695 objects degraded (12.743%), 29 pgs degraded (PG_DEGRADED) 2024-09-10T03:33:22.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:22 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 216/1695 objects degraded (12.743%), 29 pgs degraded (PG_DEGRADED) 2024-09-10T03:33:23.081 INFO:teuthology.orchestra.run.smithi130.stdout: 5 163637 11.54 MB/sec execute 272 sec latency 46.723 ms 2024-09-10T03:33:23.164 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:23 smithi130 ceph-mon[63591]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 7.0 MiB/s wr, 681 op/s; 216/1695 objects degraded (12.743%) 2024-09-10T03:33:23.279 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:33:23.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:23 smithi164 ceph-mon[49318]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 7.0 MiB/s wr, 681 op/s; 216/1695 objects degraded (12.743%) 2024-09-10T03:33:24.081 INFO:teuthology.orchestra.run.smithi130.stdout: 5 163888 11.52 MB/sec execute 273 sec latency 52.430 ms 2024-09-10T03:33:24.189 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:24 smithi130 ceph-mon[63591]: from='client.44143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:33:24.356 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:33:24.356 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (4m) 31s ago 13m 20.0M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:33:24.356 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (46s) 31s ago 13m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 8c7cfb7e87a1 2024-09-10T03:33:24.356 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (40s) 35s ago 12m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 243c094648b8 2024-09-10T03:33:24.356 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (4m) 31s ago 12m 55.7M - dad864ee21e9 72ba01ac1871 2024-09-10T03:33:24.356 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (7m) 31s ago 7m 37.1M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:33:24.356 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (7m) 35s ago 7m 15.9M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:33:24.356 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (3m) 31s ago 14m 509M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:33:24.356 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283,8765 running (2m) 35s ago 12m 453M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e4034e8e2a6d 2024-09-10T03:33:24.356 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (2m) 31s ago 14m 45.9M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 22e5cf56527a 2024-09-10T03:33:24.356 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (97s) 35s ago 12m 36.8M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 7d80bd0d15b9 2024-09-10T03:33:24.357 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (6m) 31s ago 6m 253M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:33:24.357 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (6m) 35s ago 6m 87.5M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:33:24.357 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (3m) 31s ago 12m 11.1M - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:33:24.357 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (3m) 35s ago 12m 10.6M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:33:24.357 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 starting - - - 1936M 2024-09-10T03:33:24.357 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (11m) 31s ago 11m 453M 1936M 17.2.0 e1d6a67b021e ca1ec04aecfa 2024-09-10T03:33:24.357 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (10m) 31s ago 10m 331M 1936M 17.2.0 e1d6a67b021e 5883f5d5e969 2024-09-10T03:33:24.357 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (10m) 31s ago 10m 492M 1936M 17.2.0 e1d6a67b021e b32084e1a404 2024-09-10T03:33:24.357 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (9m) 35s ago 9m 504M 2704M 17.2.0 e1d6a67b021e 3b5c9f00f64d 2024-09-10T03:33:24.357 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (9m) 35s ago 9m 559M 2704M 17.2.0 e1d6a67b021e f16357ac331e 2024-09-10T03:33:24.357 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (9m) 35s ago 9m 490M 2704M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:33:24.357 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (8m) 35s ago 8m 396M 2704M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:33:24.357 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (2m) 31s ago 12m 50.1M - 2.51.0 1d3b7f56885b f59b02fac5b3 2024-09-10T03:33:24.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:24 smithi164 ceph-mon[49318]: from='client.44143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:33:24.960 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:33:24.960 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:33:24.960 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:33:24.960 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:33:24.961 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:33:24.961 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:33:24.961 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:33:24.961 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:33:24.961 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-09-10T03:33:24.961 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:33:24.961 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:33:24.961 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:33:24.961 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:33:24.961 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:33:24.961 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-10T03:33:24.961 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 4 2024-09-10T03:33:24.961 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:33:24.962 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:33:25.081 INFO:teuthology.orchestra.run.smithi130.stdout: 5 164144 11.48 MB/sec execute 274 sec latency 85.674 ms 2024-09-10T03:33:25.207 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:25 smithi130 ceph-mon[63591]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.3 MiB/s wr, 618 op/s; 216/1695 objects degraded (12.743%) 2024-09-10T03:33:25.207 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:25 smithi130 ceph-mon[63591]: from='client.34204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:33:25.207 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:25 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/1419366771' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:33:25.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:25 smithi164 ceph-mon[49318]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.3 MiB/s wr, 618 op/s; 216/1695 objects degraded (12.743%) 2024-09-10T03:33:25.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:25 smithi164 ceph-mon[49318]: from='client.34204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:33:25.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:25 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/1419366771' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:33:25.604 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:33:25.604 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:33:25.604 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:33:25.605 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:33:25.605 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-09-10T03:33:25.605 INFO:teuthology.orchestra.run.smithi130.stdout: "mon", 2024-09-10T03:33:25.605 INFO:teuthology.orchestra.run.smithi130.stdout: "crash", 2024-09-10T03:33:25.605 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-09-10T03:33:25.605 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-09-10T03:33:25.605 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "6/23 daemons upgraded", 2024-09-10T03:33:25.605 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading osd daemons", 2024-09-10T03:33:25.605 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:33:25.605 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:33:26.081 INFO:teuthology.orchestra.run.smithi130.stdout: 5 164363 11.46 MB/sec execute 275 sec latency 39.867 ms 2024-09-10T03:33:26.232 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 217/1701 objects degraded (12.757%), 29 pgs degraded 2024-09-10T03:33:26.233 INFO:teuthology.orchestra.run.smithi130.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-10T03:33:26.233 INFO:teuthology.orchestra.run.smithi130.stdout: osd.0 (root=default,host=smithi130) is down 2024-09-10T03:33:26.233 INFO:teuthology.orchestra.run.smithi130.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 217/1701 objects degraded (12.757%), 29 pgs degraded 2024-09-10T03:33:26.233 INFO:teuthology.orchestra.run.smithi130.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-10T03:33:26.233 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-09-10T03:33:26.233 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-09-10T03:33:26.233 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-10T03:33:26.233 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-09-10T03:33:26.233 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-09-10T03:33:26.233 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-10T03:33:26.233 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-09-10T03:33:26.234 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-09-10T03:33:26.234 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-09-10T03:33:26.234 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-09-10T03:33:26.234 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-09-10T03:33:26.234 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-09-10T03:33:26.234 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-09-10T03:33:26.234 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.1f is active+undersized+degraded, acting [2,4] 2024-09-10T03:33:26.234 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-10T03:33:26.234 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-10T03:33:26.234 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-10T03:33:26.234 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-10T03:33:26.234 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-10T03:33:26.234 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-10T03:33:26.234 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-10T03:33:26.235 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-10T03:33:26.235 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-10T03:33:26.235 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-10T03:33:26.235 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-10T03:33:26.235 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-10T03:33:26.235 INFO:teuthology.orchestra.run.smithi130.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-10T03:33:26.235 INFO:teuthology.orchestra.run.smithi130.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-09-10T03:33:26.239 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:26 smithi130 ceph-mon[63591]: from='client.44151 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:33:26.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:26 smithi164 ceph-mon[49318]: from='client.44151 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:33:27.081 INFO:teuthology.orchestra.run.smithi130.stdout: 5 164693 11.45 MB/sec execute 276 sec latency 48.947 ms 2024-09-10T03:33:27.261 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:27 smithi130 ceph-mon[63591]: from='client.44159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:33:27.261 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:27 smithi130 ceph-mon[63591]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 323 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 659 op/s; 217/1701 objects degraded (12.757%) 2024-09-10T03:33:27.261 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:27 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/519657187' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:33:27.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:27 smithi164 ceph-mon[49318]: from='client.44159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:33:27.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:27 smithi164 ceph-mon[49318]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 323 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 659 op/s; 217/1701 objects degraded (12.757%) 2024-09-10T03:33:27.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:27 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/519657187' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:33:28.081 INFO:teuthology.orchestra.run.smithi130.stdout: 5 164985 11.44 MB/sec execute 277 sec latency 76.330 ms 2024-09-10T03:33:28.554 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:28 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:28.555 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:28 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:28.555 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:28 smithi130 ceph-mon[63591]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.1 MiB/s wr, 495 op/s; 217/1713 objects degraded (12.668%) 2024-09-10T03:33:28.555 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:28 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:28.555 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:28 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:28.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:28 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:28.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:28 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:28.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:28 smithi164 ceph-mon[49318]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.1 MiB/s wr, 495 op/s; 217/1713 objects degraded (12.668%) 2024-09-10T03:33:28.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:28 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:28.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:28 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:29.058 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:28 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0[68339]: 2024-09-10T03:33:28.905+0000 7f77b3519640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-10T03:33:29.081 INFO:teuthology.orchestra.run.smithi130.stdout: 5 165407 11.42 MB/sec execute 278 sec latency 47.303 ms 2024-09-10T03:33:29.587 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:29 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 217/1713 objects degraded (12.668%), 29 pgs degraded (PG_DEGRADED) 2024-09-10T03:33:29.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:29 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 217/1713 objects degraded (12.668%), 29 pgs degraded (PG_DEGRADED) 2024-09-10T03:33:30.081 INFO:teuthology.orchestra.run.smithi130.stdout: 5 165710 11.41 MB/sec execute 279 sec latency 70.232 ms 2024-09-10T03:33:30.106 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:29 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0[68339]: 2024-09-10T03:33:29.993+0000 7f77b3519640 -1 osd.0 73 log_to_monitors true 2024-09-10T03:33:30.625 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:30 smithi130 ceph-mon[63591]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.8 MiB/s wr, 479 op/s; 217/1713 objects degraded (12.668%) 2024-09-10T03:33:30.625 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:30 smithi130 ceph-mon[63591]: from='osd.0 [v2:172.21.15.130:6802/2328123089,v1:172.21.15.130:6803/2328123089]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-10T03:33:30.625 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:30.625 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:33:30.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:30 smithi164 ceph-mon[49318]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.8 MiB/s wr, 479 op/s; 217/1713 objects degraded (12.668%) 2024-09-10T03:33:30.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:30 smithi164 ceph-mon[49318]: from='osd.0 [v2:172.21.15.130:6802/2328123089,v1:172.21.15.130:6803/2328123089]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-10T03:33:30.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:30.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:33:31.082 INFO:teuthology.orchestra.run.smithi130.stdout: 5 165922 11.38 MB/sec execute 280 sec latency 120.981 ms 2024-09-10T03:33:31.131 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:33:30 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0[68339]: 2024-09-10T03:33:30.908+0000 7f77a72bb640 -1 osd.0 73 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-10T03:33:31.651 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:31 smithi130 ceph-mon[63591]: from='osd.0 [v2:172.21.15.130:6802/2328123089,v1:172.21.15.130:6803/2328123089]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-10T03:33:31.651 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:31 smithi130 ceph-mon[63591]: osdmap e76: 8 total, 7 up, 8 in 2024-09-10T03:33:31.652 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:31 smithi130 ceph-mon[63591]: from='osd.0 [v2:172.21.15.130:6802/2328123089,v1:172.21.15.130:6803/2328123089]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-09-10T03:33:31.652 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:31 smithi130 ceph-mon[63591]: from='osd.0 [v2:172.21.15.130:6802/2328123089,v1:172.21.15.130:6803/2328123089]' entity='osd.0' 2024-09-10T03:33:31.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:31 smithi164 ceph-mon[49318]: from='osd.0 [v2:172.21.15.130:6802/2328123089,v1:172.21.15.130:6803/2328123089]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-10T03:33:31.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:31 smithi164 ceph-mon[49318]: osdmap e76: 8 total, 7 up, 8 in 2024-09-10T03:33:31.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:31 smithi164 ceph-mon[49318]: from='osd.0 [v2:172.21.15.130:6802/2328123089,v1:172.21.15.130:6803/2328123089]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-09-10T03:33:31.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:31 smithi164 ceph-mon[49318]: from='osd.0 [v2:172.21.15.130:6802/2328123089,v1:172.21.15.130:6803/2328123089]' entity='osd.0' 2024-09-10T03:33:32.081 INFO:teuthology.orchestra.run.smithi130.stdout: 5 166137 11.38 MB/sec execute 281 sec latency 233.983 ms 2024-09-10T03:33:32.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:32 smithi164 ceph-mon[49318]: pgmap v67: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 323 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.1 MiB/s wr, 440 op/s; 206/1629 objects degraded (12.646%) 2024-09-10T03:33:32.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:32 smithi164 ceph-mon[49318]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-10T03:33:32.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:32 smithi164 ceph-mon[49318]: osd.0 [v2:172.21.15.130:6802/2328123089,v1:172.21.15.130:6803/2328123089] boot 2024-09-10T03:33:32.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:32 smithi164 ceph-mon[49318]: osdmap e77: 8 total, 8 up, 8 in 2024-09-10T03:33:32.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:32 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-10T03:33:32.740 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:32 smithi130 ceph-mon[63591]: pgmap v67: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 323 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.1 MiB/s wr, 440 op/s; 206/1629 objects degraded (12.646%) 2024-09-10T03:33:32.741 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:32 smithi130 ceph-mon[63591]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-10T03:33:32.741 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:32 smithi130 ceph-mon[63591]: osd.0 [v2:172.21.15.130:6802/2328123089,v1:172.21.15.130:6803/2328123089] boot 2024-09-10T03:33:32.741 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:32 smithi130 ceph-mon[63591]: osdmap e77: 8 total, 8 up, 8 in 2024-09-10T03:33:32.741 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:32 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-10T03:33:33.081 INFO:teuthology.orchestra.run.smithi130.stdout: 5 166137 11.34 MB/sec execute 282 sec latency 1234.009 ms 2024-09-10T03:33:34.081 INFO:teuthology.orchestra.run.smithi130.stdout: 5 166144 11.30 MB/sec execute 283 sec latency 2161.566 ms 2024-09-10T03:33:34.127 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:33 smithi130 ceph-mon[63591]: osdmap e78: 8 total, 8 up, 8 in 2024-09-10T03:33:34.127 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:33 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:34.127 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:33 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:34.127 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:33 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:33:34.128 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:33 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:33:34.128 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:33 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:34.128 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:33 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:34.128 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:33 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:33:34.128 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:33 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:33:34.128 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:33 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:33:34.129 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:33 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:33:34.129 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:33 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-10T03:33:34.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:33 smithi164 ceph-mon[49318]: osdmap e78: 8 total, 8 up, 8 in 2024-09-10T03:33:34.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:33 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:34.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:33 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:34.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:33 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:33:34.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:33 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:33:34.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:33 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:34.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:33 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:34.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:33 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:33:34.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:33 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:33:34.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:33 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:33:34.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:33 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:33:34.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:33 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-10T03:33:34.903 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:34 smithi130 ceph-mon[63591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-10T03:33:34.903 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:34 smithi130 ceph-mon[63591]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-10T03:33:34.903 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:34 smithi130 ceph-mon[63591]: pgmap v70: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 323 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.4 MiB/s wr, 390 op/s; 206/1629 objects degraded (12.646%) 2024-09-10T03:33:34.903 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:34 smithi130 ceph-mon[63591]: osdmap e79: 8 total, 8 up, 8 in 2024-09-10T03:33:35.084 INFO:teuthology.orchestra.run.smithi130.stdout: 5 166476 11.30 MB/sec execute 284 sec latency 44.542 ms 2024-09-10T03:33:35.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:34 smithi164 ceph-mon[49318]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-10T03:33:35.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:34 smithi164 ceph-mon[49318]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-10T03:33:35.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:34 smithi164 ceph-mon[49318]: pgmap v70: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 323 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.4 MiB/s wr, 390 op/s; 206/1629 objects degraded (12.646%) 2024-09-10T03:33:35.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:34 smithi164 ceph-mon[49318]: osdmap e79: 8 total, 8 up, 8 in 2024-09-10T03:33:35.927 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:35 smithi130 ceph-mon[63591]: osdmap e80: 8 total, 8 up, 8 in 2024-09-10T03:33:35.927 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:35 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 206/1629 objects degraded (12.646%), 29 pgs degraded (PG_DEGRADED) 2024-09-10T03:33:36.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 166745 11.29 MB/sec execute 285 sec latency 826.029 ms 2024-09-10T03:33:36.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:35 smithi164 ceph-mon[49318]: osdmap e80: 8 total, 8 up, 8 in 2024-09-10T03:33:36.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:35 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 206/1629 objects degraded (12.646%), 29 pgs degraded (PG_DEGRADED) 2024-09-10T03:33:36.949 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:36 smithi130 ceph-mon[63591]: pgmap v73: 97 pgs: 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 4 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized, 3 active+undersized+degraded, 81 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.5 MiB/s wr, 270 op/s; 187/1653 objects degraded (11.313%); 852 KiB/s, 27 objects/s recovering 2024-09-10T03:33:36.949 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:36 smithi130 ceph-mon[63591]: osdmap e81: 8 total, 8 up, 8 in 2024-09-10T03:33:37.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 166963 11.29 MB/sec execute 286 sec latency 153.297 ms 2024-09-10T03:33:37.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:36 smithi164 ceph-mon[49318]: pgmap v73: 97 pgs: 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 4 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized, 3 active+undersized+degraded, 81 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.5 MiB/s wr, 270 op/s; 187/1653 objects degraded (11.313%); 852 KiB/s, 27 objects/s recovering 2024-09-10T03:33:37.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:36 smithi164 ceph-mon[49318]: osdmap e81: 8 total, 8 up, 8 in 2024-09-10T03:33:37.985 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:37 smithi130 ceph-mon[63591]: osdmap e82: 8 total, 8 up, 8 in 2024-09-10T03:33:38.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 167048 11.25 MB/sec execute 287 sec latency 989.158 ms 2024-09-10T03:33:38.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:37 smithi164 ceph-mon[49318]: osdmap e82: 8 total, 8 up, 8 in 2024-09-10T03:33:38.993 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:38 smithi130 ceph-mon[63591]: pgmap v76: 97 pgs: 1 active+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 4 peering, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.7 MiB/s wr, 285 op/s; 173/1641 objects degraded (10.542%); 852 KiB/s, 34 objects/s recovering 2024-09-10T03:33:38.993 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:38 smithi130 ceph-mon[63591]: osdmap e83: 8 total, 8 up, 8 in 2024-09-10T03:33:39.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 167306 11.23 MB/sec execute 288 sec latency 54.583 ms 2024-09-10T03:33:39.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:38 smithi164 ceph-mon[49318]: pgmap v76: 97 pgs: 1 active+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 4 peering, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.7 MiB/s wr, 285 op/s; 173/1641 objects degraded (10.542%); 852 KiB/s, 34 objects/s recovering 2024-09-10T03:33:39.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:38 smithi164 ceph-mon[49318]: osdmap e83: 8 total, 8 up, 8 in 2024-09-10T03:33:40.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 167663 11.22 MB/sec execute 289 sec latency 81.295 ms 2024-09-10T03:33:41.021 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:40 smithi130 ceph-mon[63591]: pgmap v78: 97 pgs: 1 active+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 4 peering, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 3.0 MiB/s wr, 236 op/s; 173/1641 objects degraded (10.542%); 706 KiB/s, 28 objects/s recovering 2024-09-10T03:33:41.022 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:40 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 173/1641 objects degraded (10.542%), 8 pgs degraded (PG_DEGRADED) 2024-09-10T03:33:41.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 167994 11.21 MB/sec execute 290 sec latency 36.661 ms 2024-09-10T03:33:41.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:40 smithi164 ceph-mon[49318]: pgmap v78: 97 pgs: 1 active+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 4 peering, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 3.0 MiB/s wr, 236 op/s; 173/1641 objects degraded (10.542%); 706 KiB/s, 28 objects/s recovering 2024-09-10T03:33:41.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:40 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 173/1641 objects degraded (10.542%), 8 pgs degraded (PG_DEGRADED) 2024-09-10T03:33:42.037 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:41 smithi130 ceph-mon[63591]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 173/1641 objects degraded (10.542%), 8 pgs degraded) 2024-09-10T03:33:42.037 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:41 smithi130 ceph-mon[63591]: Cluster is now healthy 2024-09-10T03:33:42.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 168354 11.20 MB/sec execute 291 sec latency 61.710 ms 2024-09-10T03:33:42.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:41 smithi164 ceph-mon[49318]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 173/1641 objects degraded (10.542%), 8 pgs degraded) 2024-09-10T03:33:42.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:41 smithi164 ceph-mon[49318]: Cluster is now healthy 2024-09-10T03:33:43.054 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:42 smithi130 ceph-mon[63591]: pgmap v79: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.1 MiB/s wr, 316 op/s; 14 B/s, 39 objects/s recovering 2024-09-10T03:33:43.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 168737 11.19 MB/sec execute 292 sec latency 53.629 ms 2024-09-10T03:33:43.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:42 smithi164 ceph-mon[49318]: pgmap v79: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.1 MiB/s wr, 316 op/s; 14 B/s, 39 objects/s recovering 2024-09-10T03:33:44.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 169048 11.18 MB/sec execute 293 sec latency 44.164 ms 2024-09-10T03:33:45.081 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:44 smithi130 ceph-mon[63591]: pgmap v80: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 2.3 MiB/s wr, 235 op/s; 0 B/s, 26 objects/s recovering 2024-09-10T03:33:45.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 169359 11.18 MB/sec execute 294 sec latency 50.383 ms 2024-09-10T03:33:45.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:44 smithi164 ceph-mon[49318]: pgmap v80: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 2.3 MiB/s wr, 235 op/s; 0 B/s, 26 objects/s recovering 2024-09-10T03:33:46.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 169681 11.17 MB/sec execute 295 sec latency 52.776 ms 2024-09-10T03:33:46.115 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:46 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:46.116 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:46 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:33:46.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:46 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:46.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:46 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:33:47.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 170079 11.18 MB/sec execute 296 sec latency 44.909 ms 2024-09-10T03:33:47.159 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:47 smithi130 ceph-mon[63591]: pgmap v81: 97 pgs: 97 active+clean; 327 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.0 MiB/s wr, 364 op/s; 0 B/s, 23 objects/s recovering 2024-09-10T03:33:47.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:47 smithi164 ceph-mon[49318]: pgmap v81: 97 pgs: 97 active+clean; 327 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.0 MiB/s wr, 364 op/s; 0 B/s, 23 objects/s recovering 2024-09-10T03:33:48.088 INFO:teuthology.orchestra.run.smithi130.stdout: 5 170328 11.16 MB/sec execute 297 sec latency 50.435 ms 2024-09-10T03:33:49.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 170598 11.16 MB/sec execute 298 sec latency 45.181 ms 2024-09-10T03:33:49.192 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:49 smithi130 ceph-mon[63591]: pgmap v82: 97 pgs: 97 active+clean; 326 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.1 MiB/s wr, 434 op/s; 0 B/s, 20 objects/s recovering 2024-09-10T03:33:49.192 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:49 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-10T03:33:49.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:49 smithi164 ceph-mon[49318]: pgmap v82: 97 pgs: 97 active+clean; 326 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.1 MiB/s wr, 434 op/s; 0 B/s, 20 objects/s recovering 2024-09-10T03:33:49.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:49 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-10T03:33:50.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 170962 11.14 MB/sec execute 299 sec latency 44.584 ms 2024-09-10T03:33:50.211 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:50 smithi130 ceph-mon[63591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-10T03:33:50.211 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:50 smithi130 ceph-mon[63591]: Upgrade: osd.1 is safe to restart 2024-09-10T03:33:50.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:50 smithi164 ceph-mon[49318]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-10T03:33:50.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:50 smithi164 ceph-mon[49318]: Upgrade: osd.1 is safe to restart 2024-09-10T03:33:51.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 171361 11.13 MB/sec execute 300 sec latency 57.399 ms 2024-09-10T03:33:51.228 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:51 smithi130 ceph-mon[63591]: pgmap v83: 97 pgs: 97 active+clean; 326 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.4 MiB/s wr, 367 op/s; 0 B/s, 17 objects/s recovering 2024-09-10T03:33:51.228 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:51 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:51.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:51 smithi164 ceph-mon[49318]: pgmap v83: 97 pgs: 97 active+clean; 326 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.4 MiB/s wr, 367 op/s; 0 B/s, 17 objects/s recovering 2024-09-10T03:33:51.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:51 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:52.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 171758 11.14 MB/sec execute 301 sec latency 44.803 ms 2024-09-10T03:33:52.233 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:52 smithi130 ceph-mon[63591]: Upgrade: Updating osd.1 2024-09-10T03:33:52.233 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:52 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:52.233 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:52 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-10T03:33:52.233 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:52 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:33:52.233 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:52 smithi130 ceph-mon[63591]: Deploying daemon osd.1 on smithi130 2024-09-10T03:33:52.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:52 smithi164 ceph-mon[49318]: Upgrade: Updating osd.1 2024-09-10T03:33:52.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:52 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:33:52.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:52 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-10T03:33:52.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:52 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:33:52.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:52 smithi164 ceph-mon[49318]: Deploying daemon osd.1 on smithi130 2024-09-10T03:33:53.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 172229 11.13 MB/sec execute 302 sec latency 68.262 ms 2024-09-10T03:33:53.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:53 smithi130 ceph-mon[63591]: pgmap v84: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 563 op/s; 0 B/s, 17 objects/s recovering 2024-09-10T03:33:53.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:53 smithi164 ceph-mon[49318]: pgmap v84: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 563 op/s; 0 B/s, 17 objects/s recovering 2024-09-10T03:33:54.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 172560 11.12 MB/sec execute 303 sec latency 66.080 ms 2024-09-10T03:33:55.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 172895 11.12 MB/sec execute 304 sec latency 58.601 ms 2024-09-10T03:33:55.334 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:55 smithi130 ceph-mon[63591]: pgmap v85: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 410 op/s 2024-09-10T03:33:55.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:55 smithi164 ceph-mon[49318]: pgmap v85: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 410 op/s 2024-09-10T03:33:56.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 173261 11.11 MB/sec execute 305 sec latency 47.462 ms 2024-09-10T03:33:56.861 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:33:57.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 173673 11.13 MB/sec execute 306 sec latency 48.175 ms 2024-09-10T03:33:57.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:57 smithi130 ceph-mon[63591]: pgmap v86: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.9 MiB/s wr, 584 op/s 2024-09-10T03:33:57.390 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:33:57 smithi130 systemd[1]: Stopping Ceph osd.1 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:33:57.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:57 smithi164 ceph-mon[49318]: pgmap v86: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.9 MiB/s wr, 584 op/s 2024-09-10T03:33:57.896 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:33:57 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1[38063]: 2024-09-10T03:33:57.726+0000 7fd543274700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:33:57.896 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:33:57 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1[38063]: 2024-09-10T03:33:57.726+0000 7fd543274700 -1 osd.1 83 *** Got signal Terminated *** 2024-09-10T03:33:57.896 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:33:57 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1[38063]: 2024-09-10T03:33:57.726+0000 7fd543274700 -1 osd.1 83 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-10T03:33:57.961 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:33:57.961 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (4m) 30s ago 13m 19.1M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:33:57.961 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (79s) 30s ago 13m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 8c7cfb7e87a1 2024-09-10T03:33:57.961 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (73s) 69s ago 13m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 243c094648b8 2024-09-10T03:33:57.961 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (4m) 30s ago 13m 55.1M - dad864ee21e9 72ba01ac1871 2024-09-10T03:33:57.961 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (7m) 30s ago 7m 37.8M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:33:57.961 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (7m) 69s ago 7m 15.9M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:33:57.961 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (3m) 30s ago 14m 511M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:33:57.962 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283,8765 running (2m) 69s ago 13m 453M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e4034e8e2a6d 2024-09-10T03:33:57.962 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (2m) 30s ago 15m 48.0M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 22e5cf56527a 2024-09-10T03:33:57.962 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (2m) 69s ago 12m 36.8M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 7d80bd0d15b9 2024-09-10T03:33:57.962 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (7m) 30s ago 7m 250M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:33:57.962 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (7m) 69s ago 7m 87.5M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:33:57.962 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (4m) 30s ago 13m 11.2M - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:33:57.962 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (3m) 69s ago 12m 10.6M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:33:57.962 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (40s) 30s ago 12m 14.5M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b de3cbff0098b 2024-09-10T03:33:57.962 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (11m) 30s ago 11m 485M 1936M 17.2.0 e1d6a67b021e ca1ec04aecfa 2024-09-10T03:33:57.962 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (11m) 30s ago 11m 387M 1936M 17.2.0 e1d6a67b021e 5883f5d5e969 2024-09-10T03:33:57.962 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (10m) 30s ago 10m 514M 1936M 17.2.0 e1d6a67b021e b32084e1a404 2024-09-10T03:33:57.962 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (10m) 69s ago 10m 504M 2704M 17.2.0 e1d6a67b021e 3b5c9f00f64d 2024-09-10T03:33:57.962 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (9m) 69s ago 9m 559M 2704M 17.2.0 e1d6a67b021e f16357ac331e 2024-09-10T03:33:57.963 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (9m) 69s ago 9m 490M 2704M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:33:57.963 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (9m) 69s ago 9m 396M 2704M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:33:57.963 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (3m) 30s ago 12m 50.1M - 2.51.0 1d3b7f56885b f59b02fac5b3 2024-09-10T03:33:58.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 173965 11.12 MB/sec execute 307 sec latency 58.223 ms 2024-09-10T03:33:58.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:58 smithi164 ceph-mon[49318]: from='client.44169 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:33:58.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:58 smithi164 ceph-mon[49318]: osd.1 marked itself down and dead 2024-09-10T03:33:58.492 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:58 smithi130 ceph-mon[63591]: from='client.44169 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:33:58.492 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:58 smithi130 ceph-mon[63591]: osd.1 marked itself down and dead 2024-09-10T03:33:58.492 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:33:58.492 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:33:58.492 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:33:58.492 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:33:58.492 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:33:58.492 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:33:58.493 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:33:58.493 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:33:58.493 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-10T03:33:58.493 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 1 2024-09-10T03:33:58.493 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:33:58.493 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:33:58.493 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:33:58.493 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:33:58.493 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:33:58.493 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-10T03:33:58.493 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 5 2024-09-10T03:33:58.493 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:33:58.493 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:33:58.863 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:33:58.864 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:33:58.864 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:33:58.864 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:33:58.864 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-09-10T03:33:58.864 INFO:teuthology.orchestra.run.smithi130.stdout: "mon", 2024-09-10T03:33:58.864 INFO:teuthology.orchestra.run.smithi130.stdout: "crash", 2024-09-10T03:33:58.864 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-09-10T03:33:58.864 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-09-10T03:33:58.864 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "7/23 daemons upgraded", 2024-09-10T03:33:58.864 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading osd daemons", 2024-09-10T03:33:58.864 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:33:58.864 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:33:59.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 174016 11.10 MB/sec execute 308 sec latency 889.059 ms 2024-09-10T03:33:59.168 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:33:58 smithi130 podman[70269]: 2024-09-10 03:33:58.848791105 +0000 UTC m=+1.347286597 container died ca1ec04aecfaa21f43137761c44234462d70b0857bca9bf5a70b3c519cc2d4a2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, release=754, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, distribution-scope=public, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., version=8, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-10T03:33:59.343 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_WARN 1 osds down 2024-09-10T03:33:59.343 INFO:teuthology.orchestra.run.smithi130.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-10T03:33:59.343 INFO:teuthology.orchestra.run.smithi130.stdout: osd.1 (root=default,host=smithi130) is down 2024-09-10T03:33:59.418 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:59 smithi130 ceph-mon[63591]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:33:59.418 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:59 smithi130 ceph-mon[63591]: pgmap v87: 97 pgs: 97 active+clean; 327 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.8 MiB/s wr, 526 op/s 2024-09-10T03:33:59.418 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:59 smithi130 ceph-mon[63591]: from='client.44173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:33:59.418 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:59 smithi130 ceph-mon[63591]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-10T03:33:59.418 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:59 smithi130 ceph-mon[63591]: osdmap e84: 8 total, 7 up, 8 in 2024-09-10T03:33:59.418 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:33:59 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/3322441154' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:33:59.418 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:33:59 smithi130 podman[70269]: 2024-09-10 03:33:59.292834489 +0000 UTC m=+1.791329980 container cleanup ca1ec04aecfaa21f43137761c44234462d70b0857bca9bf5a70b3c519cc2d4a2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, distribution-scope=public, name=centos-stream, GIT_CLEAN=True, RELEASE=HEAD, vendor=Red Hat, Inc., architecture=x86_64, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, release=754, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-10T03:33:59.418 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:33:59 smithi130 bash[70269]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1 2024-09-10T03:33:59.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:59 smithi164 ceph-mon[49318]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:33:59.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:59 smithi164 ceph-mon[49318]: pgmap v87: 97 pgs: 97 active+clean; 327 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.8 MiB/s wr, 526 op/s 2024-09-10T03:33:59.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:59 smithi164 ceph-mon[49318]: from='client.44173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:33:59.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:59 smithi164 ceph-mon[49318]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-10T03:33:59.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:59 smithi164 ceph-mon[49318]: osdmap e84: 8 total, 7 up, 8 in 2024-09-10T03:33:59.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:33:59 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/3322441154' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:33:59.938 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:33:59 smithi130 podman[70348]: 2024-09-10 03:33:59.688268506 +0000 UTC m=+0.832783534 container remove ca1ec04aecfaa21f43137761c44234462d70b0857bca9bf5a70b3c519cc2d4a2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, RELEASE=HEAD, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, version=8, distribution-scope=public, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8) 2024-09-10T03:34:00.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 174306 11.09 MB/sec execute 309 sec latency 1050.522 ms 2024-09-10T03:34:00.442 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:00 smithi130 ceph-mon[63591]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:34:00.442 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:00 smithi130 ceph-mon[63591]: osdmap e85: 8 total, 7 up, 8 in 2024-09-10T03:34:00.442 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:00 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/2940225339' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:34:00.442 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:00.442 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:34:00.443 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:00 smithi130 podman[70457]: 2024-09-10 03:34:00.175983118 +0000 UTC m=+0.023327027 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:34:00.443 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:00 smithi130 podman[70457]: 2024-09-10 03:34:00.397341051 +0000 UTC m=+0.244684943 container create db623b99c861c80ff1a0b8d2ef4284a7490ed417e20b08ecc3263ff8e2265328 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-10T03:34:00.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:00 smithi164 ceph-mon[49318]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:34:00.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:00 smithi164 ceph-mon[49318]: osdmap e85: 8 total, 7 up, 8 in 2024-09-10T03:34:00.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:00 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/2940225339' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:34:00.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:00.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:34:00.955 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:00 smithi130 podman[70457]: 2024-09-10 03:34:00.85490442 +0000 UTC m=+0.702248323 container init db623b99c861c80ff1a0b8d2ef4284a7490ed417e20b08ecc3263ff8e2265328 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-10T03:34:00.955 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:00 smithi130 podman[70457]: 2024-09-10 03:34:00.872450053 +0000 UTC m=+0.719793953 container start db623b99c861c80ff1a0b8d2ef4284a7490ed417e20b08ecc3263ff8e2265328 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-10T03:34:01.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 174688 11.07 MB/sec execute 310 sec latency 42.841 ms 2024-09-10T03:34:01.207 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:01 smithi130 podman[70457]: 2024-09-10 03:34:01.041172906 +0000 UTC m=+0.888516804 container attach db623b99c861c80ff1a0b8d2ef4284a7490ed417e20b08ecc3263ff8e2265328 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:34:01.207 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:01 smithi130 podman[70457]: 2024-09-10 03:34:01.188615731 +0000 UTC m=+1.035959626 container died db623b99c861c80ff1a0b8d2ef4284a7490ed417e20b08ecc3263ff8e2265328 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, CEPH_POINT_RELEASE=) 2024-09-10T03:34:01.458 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:01 smithi130 ceph-mon[63591]: pgmap v90: 97 pgs: 14 stale+active+clean, 83 active+clean; 327 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.2 MiB/s wr, 345 op/s 2024-09-10T03:34:01.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:01 smithi164 ceph-mon[49318]: pgmap v90: 97 pgs: 14 stale+active+clean, 83 active+clean; 327 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.2 MiB/s wr, 345 op/s 2024-09-10T03:34:02.085 INFO:teuthology.orchestra.run.smithi130.stdout: 5 175037 11.05 MB/sec execute 311 sec latency 72.632 ms 2024-09-10T03:34:02.469 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:02 smithi130 ceph-mon[63591]: Health check failed: Degraded data redundancy: 189/1659 objects degraded (11.392%), 24 pgs degraded (PG_DEGRADED) 2024-09-10T03:34:02.469 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:02 smithi130 podman[70495]: 2024-09-10 03:34:02.466324869 +0000 UTC m=+1.267134179 container remove db623b99c861c80ff1a0b8d2ef4284a7490ed417e20b08ecc3263ff8e2265328 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-10T03:34:02.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:02 smithi164 ceph-mon[49318]: Health check failed: Degraded data redundancy: 189/1659 objects degraded (11.392%), 24 pgs degraded (PG_DEGRADED) 2024-09-10T03:34:02.729 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:02 smithi130 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-10T03:34:02.729 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:02 smithi130 systemd[1]: Stopped Ceph osd.1 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:34:02.729 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:02 smithi130 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.1.service: Consumed 2min 58.574s CPU time. 2024-09-10T03:34:02.980 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:02 smithi130 systemd[1]: Starting Ceph osd.1 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:34:03.089 INFO:teuthology.orchestra.run.smithi130.stdout: 5 175425 11.06 MB/sec execute 312 sec latency 59.107 ms 2024-09-10T03:34:03.232 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:03 smithi130 podman[70591]: 2024-09-10 03:34:03.068172741 +0000 UTC m=+0.026370671 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:34:03.484 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:03 smithi130 ceph-mon[63591]: pgmap v91: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.2 MiB/s wr, 605 op/s; 189/1659 objects degraded (11.392%) 2024-09-10T03:34:03.484 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:03 smithi130 podman[70591]: 2024-09-10 03:34:03.338130475 +0000 UTC m=+0.296328375 container create b8c7d35cd8d246b63cdd6e9e551c3353bb8584f6e702848718670760cc7ddcfa (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:34:03.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:03 smithi164 ceph-mon[49318]: pgmap v91: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.2 MiB/s wr, 605 op/s; 189/1659 objects degraded (11.392%) 2024-09-10T03:34:03.995 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:03 smithi130 podman[70591]: 2024-09-10 03:34:03.917188242 +0000 UTC m=+0.875386152 container init b8c7d35cd8d246b63cdd6e9e551c3353bb8584f6e702848718670760cc7ddcfa (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-activate, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-10T03:34:03.996 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:03 smithi130 podman[70591]: 2024-09-10 03:34:03.930322818 +0000 UTC m=+0.888520723 container start b8c7d35cd8d246b63cdd6e9e551c3353bb8584f6e702848718670760cc7ddcfa (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-activate, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:34:04.089 INFO:teuthology.orchestra.run.smithi130.stdout: 5 175909 11.05 MB/sec execute 313 sec latency 43.715 ms 2024-09-10T03:34:04.247 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:04 smithi130 podman[70591]: 2024-09-10 03:34:04.1121741 +0000 UTC m=+1.070372009 container attach b8c7d35cd8d246b63cdd6e9e551c3353bb8584f6e702848718670760cc7ddcfa (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-10T03:34:04.247 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:04 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-activate[70603]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:34:04.247 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:04 smithi130 bash[70591]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:34:04.247 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:04 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-activate[70603]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:34:04.247 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:04 smithi130 bash[70591]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:34:05.093 INFO:teuthology.orchestra.run.smithi130.stdout: 5 176252 11.04 MB/sec execute 314 sec latency 37.950 ms 2024-09-10T03:34:05.510 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:05 smithi130 ceph-mon[63591]: pgmap v92: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.5 MiB/s wr, 343 op/s; 189/1659 objects degraded (11.392%) 2024-09-10T03:34:05.510 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:05 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-activate[70603]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-10T03:34:05.510 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:05 smithi130 bash[70591]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-10T03:34:05.510 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:05 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-activate[70603]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-10T03:34:05.510 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:05 smithi130 bash[70591]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-10T03:34:05.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:05 smithi164 ceph-mon[49318]: pgmap v92: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.5 MiB/s wr, 343 op/s; 189/1659 objects degraded (11.392%) 2024-09-10T03:34:06.012 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:05 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-activate[70603]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-10T03:34:06.012 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:05 smithi130 bash[70591]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-10T03:34:06.012 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:05 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-activate[70603]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-10T03:34:06.012 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:05 smithi130 bash[70591]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-10T03:34:06.012 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:05 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-activate[70603]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-10T03:34:06.012 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:05 smithi130 bash[70591]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-10T03:34:06.012 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:05 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-activate[70603]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-10T03:34:06.013 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:05 smithi130 bash[70591]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-10T03:34:06.013 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:05 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-activate[70603]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-10T03:34:06.013 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:05 smithi130 bash[70591]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-10T03:34:06.013 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:05 smithi130 podman[70591]: 2024-09-10 03:34:05.839598528 +0000 UTC m=+2.797796439 container died b8c7d35cd8d246b63cdd6e9e551c3353bb8584f6e702848718670760cc7ddcfa (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.vendor=CentOS) 2024-09-10T03:34:06.093 INFO:teuthology.orchestra.run.smithi130.stdout: 5 176976 11.08 MB/sec execute 315 sec latency 32.134 ms 2024-09-10T03:34:07.093 INFO:teuthology.orchestra.run.smithi130.stdout: 5 177451 11.10 MB/sec execute 316 sec latency 55.429 ms 2024-09-10T03:34:07.536 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:07 smithi130 ceph-mon[63591]: pgmap v93: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.6 MiB/s wr, 648 op/s; 191/1701 objects degraded (11.229%) 2024-09-10T03:34:07.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:07 smithi164 ceph-mon[49318]: pgmap v93: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.6 MiB/s wr, 648 op/s; 191/1701 objects degraded (11.229%) 2024-09-10T03:34:08.054 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:07 smithi130 podman[70750]: 2024-09-10 03:34:07.968624113 +0000 UTC m=+2.114893444 container remove b8c7d35cd8d246b63cdd6e9e551c3353bb8584f6e702848718670760cc7ddcfa (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-10T03:34:08.093 INFO:teuthology.orchestra.run.smithi130.stdout: 5 177696 11.09 MB/sec execute 317 sec latency 62.284 ms 2024-09-10T03:34:08.305 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:08 smithi130 podman[70806]: 2024-09-10 03:34:08.095473716 +0000 UTC m=+0.019670998 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:34:08.567 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:08 smithi130 podman[70806]: 2024-09-10 03:34:08.327870128 +0000 UTC m=+0.252067401 container create f0722a6d1c90d60fb6eca54e1f5f30f2960475164a303b69600d5492867452d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-10T03:34:08.818 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:08 smithi130 podman[70806]: 2024-09-10 03:34:08.787974179 +0000 UTC m=+0.712171454 container init f0722a6d1c90d60fb6eca54e1f5f30f2960475164a303b69600d5492867452d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-10T03:34:08.818 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:08 smithi130 podman[70806]: 2024-09-10 03:34:08.803874146 +0000 UTC m=+0.728071424 container start f0722a6d1c90d60fb6eca54e1f5f30f2960475164a303b69600d5492867452d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-10T03:34:09.069 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:08 smithi130 ceph-osd[70824]: -- 172.21.15.130:0/564398697 <== mon.0 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55aa9e35a1e0 con 0x55aa9d565c00 2024-09-10T03:34:09.069 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:08 smithi130 bash[70806]: f0722a6d1c90d60fb6eca54e1f5f30f2960475164a303b69600d5492867452d6 2024-09-10T03:34:09.069 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:08 smithi130 systemd[1]: Started Ceph osd.1 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:34:09.093 INFO:teuthology.orchestra.run.smithi130.stdout: 5 178166 11.09 MB/sec execute 318 sec latency 62.648 ms 2024-09-10T03:34:09.405 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:09 smithi130 ceph-mon[63591]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.5 MiB/s wr, 544 op/s; 191/1701 objects degraded (11.229%) 2024-09-10T03:34:09.405 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:09 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:09.405 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:09 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:09.405 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:09 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:34:09.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:09 smithi164 ceph-mon[49318]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.5 MiB/s wr, 544 op/s; 191/1701 objects degraded (11.229%) 2024-09-10T03:34:09.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:09 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:09.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:09 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:09.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:09 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:34:10.093 INFO:teuthology.orchestra.run.smithi130.stdout: 5 178524 11.08 MB/sec execute 319 sec latency 102.681 ms 2024-09-10T03:34:10.434 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:10 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 191/1701 objects degraded (11.229%), 24 pgs degraded (PG_DEGRADED) 2024-09-10T03:34:10.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:10 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 191/1701 objects degraded (11.229%), 24 pgs degraded (PG_DEGRADED) 2024-09-10T03:34:10.937 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:10 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1[70820]: 2024-09-10T03:34:10.853+0000 7fc70609f640 -1 Falling back to public interface 2024-09-10T03:34:11.093 INFO:teuthology.orchestra.run.smithi130.stdout: 5 178778 11.07 MB/sec execute 320 sec latency 54.607 ms 2024-09-10T03:34:11.297 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:11 smithi130 ceph-mon[63591]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 493 op/s; 191/1701 objects degraded (11.229%) 2024-09-10T03:34:11.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:11 smithi164 ceph-mon[49318]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 493 op/s; 191/1701 objects degraded (11.229%) 2024-09-10T03:34:12.093 INFO:teuthology.orchestra.run.smithi130.stdout: 5 179122 11.06 MB/sec execute 321 sec latency 54.765 ms 2024-09-10T03:34:13.093 INFO:teuthology.orchestra.run.smithi130.stdout: 5 179541 11.04 MB/sec execute 322 sec latency 44.270 ms 2024-09-10T03:34:13.505 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:13 smithi130 ceph-mon[63591]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 7.1 MiB/s wr, 686 op/s; 203/1785 objects degraded (11.373%) 2024-09-10T03:34:13.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:13 smithi164 ceph-mon[49318]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 7.1 MiB/s wr, 686 op/s; 203/1785 objects degraded (11.373%) 2024-09-10T03:34:14.093 INFO:teuthology.orchestra.run.smithi130.stdout: 5 180105 11.07 MB/sec execute 323 sec latency 32.273 ms 2024-09-10T03:34:15.093 INFO:teuthology.orchestra.run.smithi130.stdout: 5 180841 11.10 MB/sec execute 324 sec latency 43.622 ms 2024-09-10T03:34:15.537 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:15 smithi130 ceph-mon[63591]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.2 MiB/s wr, 513 op/s; 203/1785 objects degraded (11.373%) 2024-09-10T03:34:15.537 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:15 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:15.537 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:15 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:34:15.537 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:15 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 203/1785 objects degraded (11.373%), 24 pgs degraded (PG_DEGRADED) 2024-09-10T03:34:15.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:15 smithi164 ceph-mon[49318]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.2 MiB/s wr, 513 op/s; 203/1785 objects degraded (11.373%) 2024-09-10T03:34:15.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:15 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:15.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:15 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:34:15.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:15 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 203/1785 objects degraded (11.373%), 24 pgs degraded (PG_DEGRADED) 2024-09-10T03:34:16.093 INFO:teuthology.orchestra.run.smithi130.stdout: 5 181177 11.11 MB/sec execute 325 sec latency 62.365 ms 2024-09-10T03:34:17.093 INFO:teuthology.orchestra.run.smithi130.stdout: 5 181472 11.09 MB/sec execute 326 sec latency 65.025 ms 2024-09-10T03:34:17.309 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:17 smithi130 ceph-mon[63591]: pgmap v98: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 794 op/s; 198/1815 objects degraded (10.909%) 2024-09-10T03:34:17.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:17 smithi164 ceph-mon[49318]: pgmap v98: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 794 op/s; 198/1815 objects degraded (10.909%) 2024-09-10T03:34:18.094 INFO:teuthology.orchestra.run.smithi130.stdout: 5 181723 11.08 MB/sec execute 327 sec latency 120.316 ms 2024-09-10T03:34:19.094 INFO:teuthology.orchestra.run.smithi130.stdout: 5 181995 11.06 MB/sec execute 328 sec latency 47.289 ms 2024-09-10T03:34:19.342 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:19 smithi130 ceph-mon[63591]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.7 MiB/s wr, 535 op/s; 198/1815 objects degraded (10.909%) 2024-09-10T03:34:19.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:19 smithi164 ceph-mon[49318]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.7 MiB/s wr, 535 op/s; 198/1815 objects degraded (10.909%) 2024-09-10T03:34:19.846 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:19 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1[70820]: 2024-09-10T03:34:19.659+0000 7fc70609f640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-10T03:34:20.094 INFO:teuthology.orchestra.run.smithi130.stdout: 5 182661 11.08 MB/sec execute 329 sec latency 30.571 ms 2024-09-10T03:34:20.348 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:20 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 198/1815 objects degraded (10.909%), 24 pgs degraded (PG_DEGRADED) 2024-09-10T03:34:20.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:20 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 198/1815 objects degraded (10.909%), 24 pgs degraded (PG_DEGRADED) 2024-09-10T03:34:20.855 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:20 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1[70820]: 2024-09-10T03:34:20.814+0000 7fc70609f640 -1 osd.1 83 log_to_monitors true 2024-09-10T03:34:21.094 INFO:teuthology.orchestra.run.smithi130.stdout: 5 183199 11.08 MB/sec execute 330 sec latency 33.096 ms 2024-09-10T03:34:21.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:21 smithi130 ceph-mon[63591]: pgmap v100: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.7 MiB/s wr, 536 op/s; 198/1815 objects degraded (10.909%) 2024-09-10T03:34:21.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:21 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:21.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:21 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:21.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:21 smithi130 ceph-mon[63591]: from='osd.1 [v2:172.21.15.130:6810/3778518960,v1:172.21.15.130:6811/3778518960]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-10T03:34:21.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:21 smithi164 ceph-mon[49318]: pgmap v100: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.7 MiB/s wr, 536 op/s; 198/1815 objects degraded (10.909%) 2024-09-10T03:34:21.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:21 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:21.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:21 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:21.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:21 smithi164 ceph-mon[49318]: from='osd.1 [v2:172.21.15.130:6810/3778518960,v1:172.21.15.130:6811/3778518960]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-10T03:34:21.862 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:34:21 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1[70820]: 2024-09-10T03:34:21.657+0000 7fc6fde49640 -1 osd.1 83 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-10T03:34:22.094 INFO:teuthology.orchestra.run.smithi130.stdout: 5 183724 11.10 MB/sec execute 331 sec latency 35.797 ms 2024-09-10T03:34:22.616 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:22.616 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:22.616 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:22 smithi130 ceph-mon[63591]: from='osd.1 [v2:172.21.15.130:6810/3778518960,v1:172.21.15.130:6811/3778518960]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-10T03:34:22.616 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:22 smithi130 ceph-mon[63591]: osdmap e86: 8 total, 7 up, 8 in 2024-09-10T03:34:22.616 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:22 smithi130 ceph-mon[63591]: from='osd.1 [v2:172.21.15.130:6810/3778518960,v1:172.21.15.130:6811/3778518960]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-09-10T03:34:22.616 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:22 smithi130 ceph-mon[63591]: pgmap v102: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.0 MiB/s wr, 622 op/s; 203/1767 objects degraded (11.488%) 2024-09-10T03:34:22.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:22.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:22.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:22 smithi164 ceph-mon[49318]: from='osd.1 [v2:172.21.15.130:6810/3778518960,v1:172.21.15.130:6811/3778518960]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-10T03:34:22.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:22 smithi164 ceph-mon[49318]: osdmap e86: 8 total, 7 up, 8 in 2024-09-10T03:34:22.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:22 smithi164 ceph-mon[49318]: from='osd.1 [v2:172.21.15.130:6810/3778518960,v1:172.21.15.130:6811/3778518960]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-09-10T03:34:22.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:22 smithi164 ceph-mon[49318]: pgmap v102: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.0 MiB/s wr, 622 op/s; 203/1767 objects degraded (11.488%) 2024-09-10T03:34:23.094 INFO:teuthology.orchestra.run.smithi130.stdout: 5 184045 11.08 MB/sec execute 332 sec latency 459.956 ms 2024-09-10T03:34:23.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:23 smithi164 ceph-mon[49318]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-10T03:34:23.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:23 smithi164 ceph-mon[49318]: osd.1 [v2:172.21.15.130:6810/3778518960,v1:172.21.15.130:6811/3778518960] boot 2024-09-10T03:34:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:23 smithi164 ceph-mon[49318]: osdmap e87: 8 total, 8 up, 8 in 2024-09-10T03:34:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:23 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-10T03:34:23.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:23 smithi130 ceph-mon[63591]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-10T03:34:23.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:23 smithi130 ceph-mon[63591]: osd.1 [v2:172.21.15.130:6810/3778518960,v1:172.21.15.130:6811/3778518960] boot 2024-09-10T03:34:23.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:23 smithi130 ceph-mon[63591]: osdmap e87: 8 total, 8 up, 8 in 2024-09-10T03:34:23.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:23 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-10T03:34:24.094 INFO:teuthology.orchestra.run.smithi130.stdout: 5 184045 11.05 MB/sec execute 333 sec latency 1459.985 ms 2024-09-10T03:34:24.865 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:24 smithi130 ceph-mon[63591]: osdmap e88: 8 total, 8 up, 8 in 2024-09-10T03:34:24.865 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:24 smithi130 ceph-mon[63591]: pgmap v105: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.2 MiB/s wr, 475 op/s; 203/1767 objects degraded (11.488%) 2024-09-10T03:34:24.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:24 smithi164 ceph-mon[49318]: osdmap e88: 8 total, 8 up, 8 in 2024-09-10T03:34:24.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:24 smithi164 ceph-mon[49318]: pgmap v105: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.2 MiB/s wr, 475 op/s; 203/1767 objects degraded (11.488%) 2024-09-10T03:34:25.094 INFO:teuthology.orchestra.run.smithi130.stdout: 5 184163 11.03 MB/sec execute 334 sec latency 2239.429 ms 2024-09-10T03:34:25.626 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:25 smithi130 ceph-mon[63591]: osdmap e89: 8 total, 8 up, 8 in 2024-09-10T03:34:25.626 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:25 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 203/1767 objects degraded (11.488%), 24 pgs degraded (PG_DEGRADED) 2024-09-10T03:34:25.626 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:25 smithi130 ceph-mon[63591]: osdmap e90: 8 total, 8 up, 8 in 2024-09-10T03:34:25.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:25 smithi164 ceph-mon[49318]: osdmap e89: 8 total, 8 up, 8 in 2024-09-10T03:34:25.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:25 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 203/1767 objects degraded (11.488%), 24 pgs degraded (PG_DEGRADED) 2024-09-10T03:34:25.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:25 smithi164 ceph-mon[49318]: osdmap e90: 8 total, 8 up, 8 in 2024-09-10T03:34:26.094 INFO:teuthology.orchestra.run.smithi130.stdout: 5 184528 11.04 MB/sec execute 335 sec latency 306.461 ms 2024-09-10T03:34:27.094 INFO:teuthology.orchestra.run.smithi130.stdout: 5 184693 11.02 MB/sec execute 336 sec latency 680.219 ms 2024-09-10T03:34:27.213 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:27 smithi130 ceph-mon[63591]: pgmap v108: 97 pgs: 3 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized, 7 active+undersized+degraded, 74 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.7 MiB/s wr, 527 op/s; 206/1566 objects degraded (13.155%); 1.1 MiB/s, 39 objects/s recovering 2024-09-10T03:34:27.213 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:27 smithi130 ceph-mon[63591]: osdmap e91: 8 total, 8 up, 8 in 2024-09-10T03:34:27.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:27 smithi164 ceph-mon[49318]: pgmap v108: 97 pgs: 3 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized, 7 active+undersized+degraded, 74 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.7 MiB/s wr, 527 op/s; 206/1566 objects degraded (13.155%); 1.1 MiB/s, 39 objects/s recovering 2024-09-10T03:34:27.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:27 smithi164 ceph-mon[49318]: osdmap e91: 8 total, 8 up, 8 in 2024-09-10T03:34:28.094 INFO:teuthology.orchestra.run.smithi130.stdout: 5 185131 11.02 MB/sec execute 337 sec latency 797.200 ms 2024-09-10T03:34:28.228 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:28 smithi130 ceph-mon[63591]: osdmap e92: 8 total, 8 up, 8 in 2024-09-10T03:34:28.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:28 smithi164 ceph-mon[49318]: osdmap e92: 8 total, 8 up, 8 in 2024-09-10T03:34:29.094 INFO:teuthology.orchestra.run.smithi130.stdout: 5 185364 11.00 MB/sec execute 338 sec latency 886.723 ms 2024-09-10T03:34:29.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:29 smithi130 ceph-mon[63591]: pgmap v111: 97 pgs: 2 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.8 MiB/s wr, 534 op/s; 168/1560 objects degraded (10.769%); 2.3 MiB/s, 81 keys/s, 51 objects/s recovering 2024-09-10T03:34:29.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:29 smithi130 ceph-mon[63591]: osdmap e93: 8 total, 8 up, 8 in 2024-09-10T03:34:29.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:29 smithi164 ceph-mon[49318]: pgmap v111: 97 pgs: 2 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.8 MiB/s wr, 534 op/s; 168/1560 objects degraded (10.769%); 2.3 MiB/s, 81 keys/s, 51 objects/s recovering 2024-09-10T03:34:29.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:29 smithi164 ceph-mon[49318]: osdmap e93: 8 total, 8 up, 8 in 2024-09-10T03:34:29.862 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:34:30.094 INFO:teuthology.orchestra.run.smithi130.stdout: 5 185575 10.98 MB/sec execute 339 sec latency 1014.593 ms 2024-09-10T03:34:30.306 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:30 smithi130 ceph-mon[63591]: osdmap e94: 8 total, 8 up, 8 in 2024-09-10T03:34:30.306 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:30.306 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:30.306 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:34:30.306 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:34:30.306 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:30.306 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:30.306 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:30.306 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:34:30.307 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:30 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 168/1560 objects degraded (10.769%), 6 pgs degraded (PG_DEGRADED) 2024-09-10T03:34:30.307 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:30 smithi130 ceph-mon[63591]: osdmap e95: 8 total, 8 up, 8 in 2024-09-10T03:34:30.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:30 smithi164 ceph-mon[49318]: osdmap e94: 8 total, 8 up, 8 in 2024-09-10T03:34:30.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:30.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:30.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:34:30.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:34:30.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:30.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:30.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:30.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:34:30.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:30 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 168/1560 objects degraded (10.769%), 6 pgs degraded (PG_DEGRADED) 2024-09-10T03:34:30.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:30 smithi164 ceph-mon[49318]: osdmap e95: 8 total, 8 up, 8 in 2024-09-10T03:34:30.800 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:34:30.800 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (5m) 10s ago 14m 19.1M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:34:30.800 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (112s) 10s ago 14m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 8c7cfb7e87a1 2024-09-10T03:34:30.800 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (106s) 102s ago 13m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 243c094648b8 2024-09-10T03:34:30.800 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (5m) 10s ago 13m 55.1M - dad864ee21e9 72ba01ac1871 2024-09-10T03:34:30.800 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (8m) 10s ago 8m 37.7M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:34:30.800 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (8m) 102s ago 8m 15.9M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:34:30.800 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (4m) 10s ago 15m 515M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:34:30.800 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283,8765 running (3m) 102s ago 13m 453M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e4034e8e2a6d 2024-09-10T03:34:30.800 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (3m) 10s ago 15m 50.9M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 22e5cf56527a 2024-09-10T03:34:30.800 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (2m) 102s ago 13m 36.8M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 7d80bd0d15b9 2024-09-10T03:34:30.801 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (7m) 10s ago 7m 269M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:34:30.801 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (7m) 102s ago 7m 87.5M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:34:30.801 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (4m) 10s ago 13m 11.9M - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:34:30.801 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (4m) 102s ago 13m 10.6M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:34:30.801 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (72s) 10s ago 12m 243M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b de3cbff0098b 2024-09-10T03:34:30.801 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (22s) 10s ago 12m 12.1M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f0722a6d1c90 2024-09-10T03:34:30.801 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (11m) 10s ago 11m 433M 1936M 17.2.0 e1d6a67b021e 5883f5d5e969 2024-09-10T03:34:30.801 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (11m) 10s ago 11m 568M 1936M 17.2.0 e1d6a67b021e b32084e1a404 2024-09-10T03:34:30.801 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (10m) 102s ago 10m 504M 2704M 17.2.0 e1d6a67b021e 3b5c9f00f64d 2024-09-10T03:34:30.801 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (10m) 102s ago 10m 559M 2704M 17.2.0 e1d6a67b021e f16357ac331e 2024-09-10T03:34:30.801 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (10m) 102s ago 10m 490M 2704M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:34:30.801 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (9m) 102s ago 9m 396M 2704M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:34:30.801 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (3m) 10s ago 13m 52.3M - 2.51.0 1d3b7f56885b f59b02fac5b3 2024-09-10T03:34:31.094 INFO:teuthology.orchestra.run.smithi130.stdout: 5 185778 10.97 MB/sec execute 340 sec latency 900.853 ms 2024-09-10T03:34:31.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:31 smithi130 ceph-mon[63591]: pgmap v114: 97 pgs: 2 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 75 KiB/s wr, 6 op/s; 168/1560 objects degraded (10.769%); 1.2 MiB/s, 81 keys/s, 11 objects/s recovering 2024-09-10T03:34:31.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:31 smithi130 ceph-mon[63591]: from='client.44183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:34:31.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:31 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:34:31.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:31 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:34:31.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:31 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:34:31.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:31 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:34:31.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:31 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-10T03:34:31.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:31 smithi130 ceph-mon[63591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-10T03:34:31.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:31 smithi130 ceph-mon[63591]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-10T03:34:31.351 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:31 smithi130 ceph-mon[63591]: osdmap e96: 8 total, 8 up, 8 in 2024-09-10T03:34:31.563 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:34:31.563 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:34:31.563 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:34:31.563 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:34:31.563 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:34:31.563 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:34:31.563 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:34:31.563 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:34:31.563 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-10T03:34:31.564 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:34:31.564 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:34:31.564 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:34:31.564 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:34:31.564 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:34:31.564 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:34:31.564 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-10T03:34:31.564 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 6 2024-09-10T03:34:31.564 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:34:31.564 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:34:31.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:31 smithi164 ceph-mon[49318]: pgmap v114: 97 pgs: 2 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 75 KiB/s wr, 6 op/s; 168/1560 objects degraded (10.769%); 1.2 MiB/s, 81 keys/s, 11 objects/s recovering 2024-09-10T03:34:31.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:31 smithi164 ceph-mon[49318]: from='client.44183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:34:31.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:31 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:34:31.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:31 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:34:31.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:31 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:34:31.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:31 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:34:31.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:31 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-10T03:34:31.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:31 smithi164 ceph-mon[49318]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-10T03:34:31.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:31 smithi164 ceph-mon[49318]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-10T03:34:31.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:31 smithi164 ceph-mon[49318]: osdmap e96: 8 total, 8 up, 8 in 2024-09-10T03:34:32.029 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:34:32.029 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:34:32.029 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:34:32.029 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:34:32.030 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-09-10T03:34:32.030 INFO:teuthology.orchestra.run.smithi130.stdout: "mon", 2024-09-10T03:34:32.030 INFO:teuthology.orchestra.run.smithi130.stdout: "crash", 2024-09-10T03:34:32.030 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-09-10T03:34:32.030 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-09-10T03:34:32.030 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "8/23 daemons upgraded", 2024-09-10T03:34:32.030 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading osd daemons", 2024-09-10T03:34:32.030 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:34:32.030 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:34:32.094 INFO:teuthology.orchestra.run.smithi130.stdout: 5 186020 10.96 MB/sec execute 341 sec latency 1022.880 ms 2024-09-10T03:34:32.371 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:32 smithi130 ceph-mon[63591]: from='client.44187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:34:32.371 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:32 smithi130 ceph-mon[63591]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:34:32.371 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:32 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/2853938922' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:34:32.371 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:32 smithi130 ceph-mon[63591]: osdmap e97: 8 total, 8 up, 8 in 2024-09-10T03:34:32.549 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_WARN Degraded data redundancy: 33/1716 objects degraded (1.923%), 2 pgs degraded 2024-09-10T03:34:32.549 INFO:teuthology.orchestra.run.smithi130.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 33/1716 objects degraded (1.923%), 2 pgs degraded 2024-09-10T03:34:32.549 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.18 is active+recovery_wait+undersized+degraded+remapped, acting [3,0] 2024-09-10T03:34:32.549 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.1c is active+recovering+undersized+degraded+remapped, acting [5,4] 2024-09-10T03:34:32.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:32 smithi164 ceph-mon[49318]: from='client.44187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:34:32.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:32 smithi164 ceph-mon[49318]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:34:32.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:32 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/2853938922' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:34:32.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:32 smithi164 ceph-mon[49318]: osdmap e97: 8 total, 8 up, 8 in 2024-09-10T03:34:33.094 INFO:teuthology.orchestra.run.smithi130.stdout: 5 186266 10.94 MB/sec execute 342 sec latency 748.992 ms 2024-09-10T03:34:33.378 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:33 smithi130 ceph-mon[63591]: pgmap v117: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.2 MiB/s wr, 480 op/s; 33/1716 objects degraded (1.923%); 0 B/s, 33 objects/s recovering 2024-09-10T03:34:33.378 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:33 smithi130 ceph-mon[63591]: from='client.34270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:34:33.378 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:33 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/454068680' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:34:33.378 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:33 smithi130 ceph-mon[63591]: osdmap e98: 8 total, 8 up, 8 in 2024-09-10T03:34:33.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:33 smithi164 ceph-mon[49318]: pgmap v117: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.2 MiB/s wr, 480 op/s; 33/1716 objects degraded (1.923%); 0 B/s, 33 objects/s recovering 2024-09-10T03:34:33.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:33 smithi164 ceph-mon[49318]: from='client.34270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:34:33.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:33 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/454068680' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:34:33.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:33 smithi164 ceph-mon[49318]: osdmap e98: 8 total, 8 up, 8 in 2024-09-10T03:34:34.094 INFO:teuthology.orchestra.run.smithi130.stdout: 5 186566 10.93 MB/sec execute 343 sec latency 874.275 ms 2024-09-10T03:34:35.095 INFO:teuthology.orchestra.run.smithi130.stdout: 5 186859 10.92 MB/sec execute 344 sec latency 71.800 ms 2024-09-10T03:34:35.444 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:35 smithi130 ceph-mon[63591]: pgmap v120: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.2 MiB/s wr, 480 op/s; 33/1716 objects degraded (1.923%); 0 B/s, 33 objects/s recovering 2024-09-10T03:34:35.444 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:35 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 33/1716 objects degraded (1.923%), 2 pgs degraded (PG_DEGRADED) 2024-09-10T03:34:35.509 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:35 smithi164 ceph-mon[49318]: pgmap v120: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.2 MiB/s wr, 480 op/s; 33/1716 objects degraded (1.923%); 0 B/s, 33 objects/s recovering 2024-09-10T03:34:35.510 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:35 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 33/1716 objects degraded (1.923%), 2 pgs degraded (PG_DEGRADED) 2024-09-10T03:34:36.095 INFO:teuthology.orchestra.run.smithi130.stdout: 5 187158 10.91 MB/sec execute 345 sec latency 44.279 ms 2024-09-10T03:34:36.450 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:36 smithi130 ceph-mon[63591]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/1716 objects degraded (1.923%), 2 pgs degraded) 2024-09-10T03:34:36.450 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:36 smithi130 ceph-mon[63591]: Cluster is now healthy 2024-09-10T03:34:36.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:36 smithi164 ceph-mon[49318]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/1716 objects degraded (1.923%), 2 pgs degraded) 2024-09-10T03:34:36.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:36 smithi164 ceph-mon[49318]: Cluster is now healthy 2024-09-10T03:34:37.095 INFO:teuthology.orchestra.run.smithi130.stdout: 5 187463 10.91 MB/sec execute 346 sec latency 71.591 ms 2024-09-10T03:34:37.475 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:37 smithi130 ceph-mon[63591]: pgmap v121: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 695 op/s; 0 B/s, 31 objects/s recovering 2024-09-10T03:34:37.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:37 smithi164 ceph-mon[49318]: pgmap v121: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 695 op/s; 0 B/s, 31 objects/s recovering 2024-09-10T03:34:38.095 INFO:teuthology.orchestra.run.smithi130.stdout: 5 187827 10.91 MB/sec execute 347 sec latency 54.465 ms 2024-09-10T03:34:39.095 INFO:teuthology.orchestra.run.smithi130.stdout: 5 188263 10.92 MB/sec execute 348 sec latency 30.313 ms 2024-09-10T03:34:39.488 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:39 smithi130 ceph-mon[63591]: pgmap v122: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.0 MiB/s wr, 318 op/s; 0 B/s, 12 objects/s recovering 2024-09-10T03:34:39.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:39 smithi164 ceph-mon[49318]: pgmap v122: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.0 MiB/s wr, 318 op/s; 0 B/s, 12 objects/s recovering 2024-09-10T03:34:40.095 INFO:teuthology.orchestra.run.smithi130.stdout: 5 188661 10.91 MB/sec execute 349 sec latency 49.465 ms 2024-09-10T03:34:41.095 INFO:teuthology.orchestra.run.smithi130.stdout: 5 188966 10.90 MB/sec execute 350 sec latency 52.705 ms 2024-09-10T03:34:41.530 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:41 smithi130 ceph-mon[63591]: pgmap v123: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 2.2 MiB/s wr, 250 op/s; 0 B/s, 4 objects/s recovering 2024-09-10T03:34:41.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:41 smithi164 ceph-mon[49318]: pgmap v123: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 2.2 MiB/s wr, 250 op/s; 0 B/s, 4 objects/s recovering 2024-09-10T03:34:42.095 INFO:teuthology.orchestra.run.smithi130.stdout: 5 189195 10.89 MB/sec execute 351 sec latency 54.011 ms 2024-09-10T03:34:43.095 INFO:teuthology.orchestra.run.smithi130.stdout: 5 189487 10.88 MB/sec execute 352 sec latency 49.329 ms 2024-09-10T03:34:43.299 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:43 smithi130 ceph-mon[63591]: pgmap v124: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 484 op/s; 0 B/s, 3 objects/s recovering 2024-09-10T03:34:43.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:43 smithi164 ceph-mon[49318]: pgmap v124: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 484 op/s; 0 B/s, 3 objects/s recovering 2024-09-10T03:34:44.095 INFO:teuthology.orchestra.run.smithi130.stdout: 5 189859 10.87 MB/sec execute 353 sec latency 60.238 ms 2024-09-10T03:34:45.095 INFO:teuthology.orchestra.run.smithi130.stdout: 5 190198 10.87 MB/sec execute 354 sec latency 50.192 ms 2024-09-10T03:34:45.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:45 smithi130 ceph-mon[63591]: pgmap v125: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.4 MiB/s wr, 439 op/s; 0 B/s, 3 objects/s recovering 2024-09-10T03:34:45.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:45 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:45.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:45 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:34:45.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:45 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:45.426 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:45 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-10T03:34:45.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:45 smithi164 ceph-mon[49318]: pgmap v125: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.4 MiB/s wr, 439 op/s; 0 B/s, 3 objects/s recovering 2024-09-10T03:34:45.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:45 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:45.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:45 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:34:45.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:45 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:45.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:45 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-10T03:34:46.095 INFO:teuthology.orchestra.run.smithi130.stdout: 5 190484 10.86 MB/sec execute 355 sec latency 47.293 ms 2024-09-10T03:34:46.439 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:46 smithi130 ceph-mon[63591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-10T03:34:46.439 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:46 smithi130 ceph-mon[63591]: Upgrade: osd.2 is safe to restart 2024-09-10T03:34:46.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:46 smithi164 ceph-mon[49318]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-10T03:34:46.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:46 smithi164 ceph-mon[49318]: Upgrade: osd.2 is safe to restart 2024-09-10T03:34:47.095 INFO:teuthology.orchestra.run.smithi130.stdout: 5 190873 10.86 MB/sec execute 356 sec latency 74.577 ms 2024-09-10T03:34:47.475 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:47 smithi130 ceph-mon[63591]: pgmap v126: 97 pgs: 97 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.6 MiB/s wr, 571 op/s; 0 B/s, 2 objects/s recovering 2024-09-10T03:34:47.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:47 smithi164 ceph-mon[49318]: pgmap v126: 97 pgs: 97 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.6 MiB/s wr, 571 op/s; 0 B/s, 2 objects/s recovering 2024-09-10T03:34:48.095 INFO:teuthology.orchestra.run.smithi130.stdout: 5 191261 10.86 MB/sec execute 357 sec latency 44.873 ms 2024-09-10T03:34:49.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:48 smithi130 ceph-mon[63591]: pgmap v127: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-09-10T03:34:49.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:48 smithi130 ceph-mon[63591]: Upgrade: Updating osd.2 2024-09-10T03:34:49.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:48 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:49.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:48 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-10T03:34:49.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:48 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:34:49.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:48 smithi130 ceph-mon[63591]: Deploying daemon osd.2 on smithi130 2024-09-10T03:34:49.095 INFO:teuthology.orchestra.run.smithi130.stdout: 5 191595 10.86 MB/sec execute 358 sec latency 44.630 ms 2024-09-10T03:34:49.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:48 smithi164 ceph-mon[49318]: pgmap v127: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-09-10T03:34:49.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:48 smithi164 ceph-mon[49318]: Upgrade: Updating osd.2 2024-09-10T03:34:49.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:48 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:34:49.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:48 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-10T03:34:49.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:48 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:34:49.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:48 smithi164 ceph-mon[49318]: Deploying daemon osd.2 on smithi130 2024-09-10T03:34:50.096 INFO:teuthology.orchestra.run.smithi130.stdout: 5 191969 10.86 MB/sec execute 359 sec latency 58.760 ms 2024-09-10T03:34:51.054 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:50 smithi130 ceph-mon[63591]: pgmap v128: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 412 op/s 2024-09-10T03:34:51.095 INFO:teuthology.orchestra.run.smithi130.stdout: 5 192362 10.85 MB/sec execute 360 sec latency 66.928 ms 2024-09-10T03:34:51.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:50 smithi164 ceph-mon[49318]: pgmap v128: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 412 op/s 2024-09-10T03:34:52.096 INFO:teuthology.orchestra.run.smithi130.stdout: 5 192646 10.85 MB/sec execute 361 sec latency 43.711 ms 2024-09-10T03:34:53.096 INFO:teuthology.orchestra.run.smithi130.stdout: 5 193004 10.85 MB/sec execute 362 sec latency 53.565 ms 2024-09-10T03:34:53.101 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:52 smithi130 ceph-mon[63591]: pgmap v129: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.4 MiB/s wr, 623 op/s 2024-09-10T03:34:53.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:52 smithi164 ceph-mon[49318]: pgmap v129: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.4 MiB/s wr, 623 op/s 2024-09-10T03:34:54.096 INFO:teuthology.orchestra.run.smithi130.stdout: 5 193289 10.83 MB/sec execute 363 sec latency 69.717 ms 2024-09-10T03:34:54.933 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:54 smithi130 ceph-mon[63591]: pgmap v130: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.1 MiB/s wr, 405 op/s 2024-09-10T03:34:54.933 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:34:54 smithi130 systemd[1]: Stopping Ceph osd.2 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:34:55.096 INFO:teuthology.orchestra.run.smithi130.stdout: 5 193564 10.82 MB/sec execute 364 sec latency 56.253 ms 2024-09-10T03:34:55.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:54 smithi164 ceph-mon[49318]: pgmap v130: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.1 MiB/s wr, 405 op/s 2024-09-10T03:34:55.448 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:34:55 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2[41711]: 2024-09-10T03:34:55.206+0000 7f3874e40700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:34:55.448 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:34:55 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2[41711]: 2024-09-10T03:34:55.206+0000 7f3874e40700 -1 osd.2 98 *** Got signal Terminated *** 2024-09-10T03:34:55.448 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:34:55 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2[41711]: 2024-09-10T03:34:55.206+0000 7f3874e40700 -1 osd.2 98 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-10T03:34:56.096 INFO:teuthology.orchestra.run.smithi130.stdout: 5 193844 10.82 MB/sec execute 365 sec latency 90.096 ms 2024-09-10T03:34:56.230 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:56 smithi130 ceph-mon[63591]: osd.2 marked itself down and dead 2024-09-10T03:34:56.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:56 smithi164 ceph-mon[49318]: osd.2 marked itself down and dead 2024-09-10T03:34:57.012 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:34:56 smithi130 podman[72682]: 2024-09-10 03:34:56.910662204 +0000 UTC m=+1.987313439 container died 5883f5d5e96969b63a8cc4ad647e8952e04595b259edb8cec70bea43808ca033 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.k8s.display-name=CentOS Stream 8, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, RELEASE=HEAD, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8) 2024-09-10T03:34:57.096 INFO:teuthology.orchestra.run.smithi130.stdout: 5 194007 10.79 MB/sec execute 366 sec latency 901.242 ms 2024-09-10T03:34:57.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:57 smithi130 ceph-mon[63591]: pgmap v131: 97 pgs: 97 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 597 op/s 2024-09-10T03:34:57.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:57 smithi130 ceph-mon[63591]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-10T03:34:57.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:57 smithi130 ceph-mon[63591]: osdmap e99: 8 total, 7 up, 8 in 2024-09-10T03:34:57.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:57 smithi164 ceph-mon[49318]: pgmap v131: 97 pgs: 97 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 597 op/s 2024-09-10T03:34:57.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:57 smithi164 ceph-mon[49318]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-10T03:34:57.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:57 smithi164 ceph-mon[49318]: osdmap e99: 8 total, 7 up, 8 in 2024-09-10T03:34:58.039 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:34:57 smithi130 podman[72682]: 2024-09-10 03:34:57.792593853 +0000 UTC m=+2.869245096 container cleanup 5883f5d5e96969b63a8cc4ad647e8952e04595b259edb8cec70bea43808ca033 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, version=8, ceph=True, release=754, name=centos-stream, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_BRANCH=HEAD, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870) 2024-09-10T03:34:58.039 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:34:57 smithi130 bash[72682]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2 2024-09-10T03:34:58.096 INFO:teuthology.orchestra.run.smithi130.stdout: 5 194233 10.77 MB/sec execute 367 sec latency 1029.362 ms 2024-09-10T03:34:58.297 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:58 smithi130 ceph-mon[63591]: osdmap e100: 8 total, 7 up, 8 in 2024-09-10T03:34:58.552 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:34:58 smithi130 podman[72698]: 2024-09-10 03:34:58.346365084 +0000 UTC m=+1.438904551 container remove 5883f5d5e96969b63a8cc4ad647e8952e04595b259edb8cec70bea43808ca033 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2, maintainer=Guillaume Abrioux , version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, RELEASE=HEAD, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, vendor=Red Hat, Inc.) 2024-09-10T03:34:58.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:58 smithi164 ceph-mon[49318]: osdmap e100: 8 total, 7 up, 8 in 2024-09-10T03:34:59.056 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:34:58 smithi130 podman[72782]: 2024-09-10 03:34:58.707670914 +0000 UTC m=+0.019640992 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:34:59.096 INFO:teuthology.orchestra.run.smithi130.stdout: 5 194540 10.77 MB/sec execute 368 sec latency 49.907 ms 2024-09-10T03:34:59.307 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:34:59 smithi130 ceph-mon[63591]: pgmap v134: 97 pgs: 6 peering, 3 stale+active+clean, 88 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.2 MiB/s wr, 619 op/s 2024-09-10T03:34:59.307 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:34:59 smithi130 podman[72782]: 2024-09-10 03:34:59.148853138 +0000 UTC m=+0.460823189 container create 4d193ffe9803a5407ef27852241267cc4b7434340a65b6e247c7574f3c1dd1f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:34:59.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:34:59 smithi164 ceph-mon[49318]: pgmap v134: 97 pgs: 6 peering, 3 stale+active+clean, 88 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.2 MiB/s wr, 619 op/s 2024-09-10T03:34:59.810 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:34:59 smithi130 podman[72782]: 2024-09-10 03:34:59.735696318 +0000 UTC m=+1.047666393 container init 4d193ffe9803a5407ef27852241267cc4b7434340a65b6e247c7574f3c1dd1f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.schema-version=1.0) 2024-09-10T03:34:59.810 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:34:59 smithi130 podman[72782]: 2024-09-10 03:34:59.749708369 +0000 UTC m=+1.061678423 container start 4d193ffe9803a5407ef27852241267cc4b7434340a65b6e247c7574f3c1dd1f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-10T03:35:00.061 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:34:59 smithi130 podman[72782]: 2024-09-10 03:34:59.911302031 +0000 UTC m=+1.223272084 container attach 4d193ffe9803a5407ef27852241267cc4b7434340a65b6e247c7574f3c1dd1f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-10T03:35:00.096 INFO:teuthology.orchestra.run.smithi130.stdout: 5 194935 10.77 MB/sec execute 369 sec latency 65.697 ms 2024-09-10T03:35:00.318 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:00 smithi130 podman[72782]: 2024-09-10 03:35:00.106847759 +0000 UTC m=+1.418817816 container died 4d193ffe9803a5407ef27852241267cc4b7434340a65b6e247c7574f3c1dd1f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-10T03:35:01.077 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:01 smithi130 ceph-mon[63591]: pgmap v135: 97 pgs: 6 peering, 3 stale+active+clean, 88 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 2.8 MiB/s wr, 301 op/s 2024-09-10T03:35:01.077 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:01.077 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:35:01.096 INFO:teuthology.orchestra.run.smithi130.stdout: 5 195355 10.78 MB/sec execute 370 sec latency 35.374 ms 2024-09-10T03:35:01.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:01 smithi164 ceph-mon[49318]: pgmap v135: 97 pgs: 6 peering, 3 stale+active+clean, 88 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 2.8 MiB/s wr, 301 op/s 2024-09-10T03:35:01.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:35:01.583 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:01 smithi130 podman[72804]: 2024-09-10 03:35:01.429269611 +0000 UTC m=+1.313733180 container remove 4d193ffe9803a5407ef27852241267cc4b7434340a65b6e247c7574f3c1dd1f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True) 2024-09-10T03:35:01.583 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:01 smithi130 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-10T03:35:01.583 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:01 smithi130 systemd[1]: Stopped Ceph osd.2 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:35:01.583 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:01 smithi130 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.2.service: Consumed 2min 27.970s CPU time. 2024-09-10T03:35:01.847 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:01 smithi130 systemd[1]: Starting Ceph osd.2 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:35:02.096 INFO:teuthology.orchestra.run.smithi130.stdout: 5 195678 10.78 MB/sec execute 371 sec latency 54.554 ms 2024-09-10T03:35:02.098 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:02 smithi130 podman[72917]: 2024-09-10 03:35:01.918620721 +0000 UTC m=+0.021400609 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:35:02.098 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:02 smithi130 podman[72917]: 2024-09-10 03:35:02.056384817 +0000 UTC m=+0.159164702 container create a98c56ae10b712a2ad3239d4153f9b8473ae14aa4a5e552362143cdf972b7b55 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-09-10T03:35:02.350 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:02 smithi130 ceph-mon[63591]: Health check failed: Degraded data redundancy: 158/1668 objects degraded (9.472%), 19 pgs degraded (PG_DEGRADED) 2024-09-10T03:35:02.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:02 smithi164 ceph-mon[49318]: Health check failed: Degraded data redundancy: 158/1668 objects degraded (9.472%), 19 pgs degraded (PG_DEGRADED) 2024-09-10T03:35:02.607 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:02 smithi130 podman[72917]: 2024-09-10 03:35:02.376645902 +0000 UTC m=+0.479425813 container init a98c56ae10b712a2ad3239d4153f9b8473ae14aa4a5e552362143cdf972b7b55 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-10T03:35:02.607 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:02 smithi130 podman[72917]: 2024-09-10 03:35:02.385674646 +0000 UTC m=+0.488454540 container start a98c56ae10b712a2ad3239d4153f9b8473ae14aa4a5e552362143cdf972b7b55 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:35:02.608 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:02 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-activate[72931]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:35:02.608 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:02 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-activate[72931]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:35:02.862 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:02 smithi130 bash[72917]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:35:02.862 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:02 smithi130 bash[72917]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:35:02.862 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:02 smithi130 podman[72917]: 2024-09-10 03:35:02.685656521 +0000 UTC m=+0.788436421 container attach a98c56ae10b712a2ad3239d4153f9b8473ae14aa4a5e552362143cdf972b7b55 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-09-10T03:35:03.073 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:35:03.096 INFO:teuthology.orchestra.run.smithi130.stdout: 5 195923 10.76 MB/sec execute 372 sec latency 63.681 ms 2024-09-10T03:35:03.113 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:03 smithi130 ceph-mon[63591]: pgmap v136: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 327 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 569 op/s; 158/1668 objects degraded (9.472%) 2024-09-10T03:35:03.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:03 smithi164 ceph-mon[49318]: pgmap v136: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 327 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 569 op/s; 158/1668 objects degraded (9.472%) 2024-09-10T03:35:03.640 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:03 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-activate[72931]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-10T03:35:03.640 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:03 smithi130 bash[72917]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-10T03:35:03.640 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:03 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-activate[72931]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-10T03:35:03.640 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:03 smithi130 bash[72917]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-10T03:35:03.890 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:03 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-activate[72931]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-10T03:35:03.891 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:03 smithi130 bash[72917]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-10T03:35:03.916 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:35:03.916 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (5m) 43s ago 14m 19.1M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:35:03.916 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (2m) 43s ago 14m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 8c7cfb7e87a1 2024-09-10T03:35:03.916 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (2m) 2m ago 14m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 243c094648b8 2024-09-10T03:35:03.916 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (5m) 43s ago 14m 55.1M - dad864ee21e9 72ba01ac1871 2024-09-10T03:35:03.916 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (8m) 43s ago 8m 37.7M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:35:03.917 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (8m) 2m ago 8m 15.9M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:35:03.917 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (4m) 43s ago 15m 515M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:35:03.917 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283,8765 running (4m) 2m ago 14m 453M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e4034e8e2a6d 2024-09-10T03:35:03.917 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (3m) 43s ago 16m 50.9M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 22e5cf56527a 2024-09-10T03:35:03.917 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (3m) 2m ago 14m 36.8M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 7d80bd0d15b9 2024-09-10T03:35:03.917 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (8m) 43s ago 8m 269M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:35:03.917 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (8m) 2m ago 8m 87.5M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:35:03.917 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (5m) 43s ago 14m 11.9M - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:35:03.917 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (5m) 2m ago 13m 10.6M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:35:03.917 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (106s) 43s ago 13m 243M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b de3cbff0098b 2024-09-10T03:35:03.917 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (55s) 43s ago 12m 12.1M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f0722a6d1c90 2024-09-10T03:35:03.917 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (12m) 43s ago 12m 433M 1936M 17.2.0 e1d6a67b021e 5883f5d5e969 2024-09-10T03:35:03.917 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (11m) 43s ago 11m 568M 1936M 17.2.0 e1d6a67b021e b32084e1a404 2024-09-10T03:35:03.917 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (11m) 2m ago 11m 504M 2704M 17.2.0 e1d6a67b021e 3b5c9f00f64d 2024-09-10T03:35:03.918 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (11m) 2m ago 11m 559M 2704M 17.2.0 e1d6a67b021e f16357ac331e 2024-09-10T03:35:03.918 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (10m) 2m ago 10m 490M 2704M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:35:03.918 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (10m) 2m ago 10m 396M 2704M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:35:03.918 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (4m) 43s ago 13m 52.3M - 2.51.0 1d3b7f56885b f59b02fac5b3 2024-09-10T03:35:04.096 INFO:teuthology.orchestra.run.smithi130.stdout: 5 196195 10.76 MB/sec execute 373 sec latency 49.658 ms 2024-09-10T03:35:04.149 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:04 smithi130 ceph-mon[63591]: from='client.34278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:35:04.149 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:03 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-activate[72931]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-10T03:35:04.149 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:03 smithi130 bash[72917]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-10T03:35:04.149 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:03 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-activate[72931]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-10T03:35:04.149 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:03 smithi130 bash[72917]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-10T03:35:04.149 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:03 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-activate[72931]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-10T03:35:04.149 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:03 smithi130 bash[72917]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-10T03:35:04.150 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:03 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-activate[72931]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-10T03:35:04.150 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:03 smithi130 bash[72917]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-10T03:35:04.150 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:03 smithi130 podman[72917]: 2024-09-10 03:35:03.939481468 +0000 UTC m=+2.042261357 container died a98c56ae10b712a2ad3239d4153f9b8473ae14aa4a5e552362143cdf972b7b55 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-10T03:35:04.457 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:35:04.457 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:35:04.457 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:35:04.457 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:35:04.457 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:35:04.457 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:35:04.457 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:35:04.457 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:35:04.458 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-10T03:35:04.458 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:35:04.458 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:35:04.458 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:35:04.458 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:35:04.458 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:35:04.458 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:35:04.458 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-10T03:35:04.458 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 6 2024-09-10T03:35:04.458 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:35:04.458 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:35:04.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:04 smithi164 ceph-mon[49318]: from='client.34278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:35:04.888 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:35:04.888 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:35:04.888 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:35:04.888 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:35:04.888 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-09-10T03:35:04.888 INFO:teuthology.orchestra.run.smithi130.stdout: "mon", 2024-09-10T03:35:04.888 INFO:teuthology.orchestra.run.smithi130.stdout: "crash", 2024-09-10T03:35:04.888 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-09-10T03:35:04.888 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-09-10T03:35:04.889 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "8/23 daemons upgraded", 2024-09-10T03:35:04.889 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading osd daemons", 2024-09-10T03:35:04.889 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:35:04.889 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:35:05.096 INFO:teuthology.orchestra.run.smithi130.stdout: 5 196456 10.75 MB/sec execute 374 sec latency 40.525 ms 2024-09-10T03:35:05.164 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:05 smithi130 ceph-mon[63591]: from='client.34282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:35:05.164 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:05 smithi130 ceph-mon[63591]: pgmap v137: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 327 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 2.9 MiB/s wr, 280 op/s; 158/1668 objects degraded (9.472%) 2024-09-10T03:35:05.164 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:05 smithi130 ceph-mon[63591]: from='client.34286 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:35:05.164 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:05 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/1061544155' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:35:05.164 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:05 smithi130 podman[73153]: 2024-09-10 03:35:05.056687129 +0000 UTC m=+1.108484573 container remove a98c56ae10b712a2ad3239d4153f9b8473ae14aa4a5e552362143cdf972b7b55 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-activate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-10T03:35:05.359 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:05 smithi164 ceph-mon[49318]: from='client.34282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:35:05.359 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:05 smithi164 ceph-mon[49318]: pgmap v137: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 327 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 2.9 MiB/s wr, 280 op/s; 158/1668 objects degraded (9.472%) 2024-09-10T03:35:05.359 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:05 smithi164 ceph-mon[49318]: from='client.34286 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:35:05.360 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:05 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/1061544155' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:35:05.416 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:05 smithi130 podman[73278]: 2024-09-10 03:35:05.244831948 +0000 UTC m=+0.029640935 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:35:05.416 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:05 smithi130 podman[73278]: 2024-09-10 03:35:05.362476775 +0000 UTC m=+0.147285753 container create a871d2ccf96efe15953b07f959ddd4e758b8a8c3c0db3dc5f696512da7851e38 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-10T03:35:05.575 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 158/1668 objects degraded (9.472%), 19 pgs degraded 2024-09-10T03:35:05.575 INFO:teuthology.orchestra.run.smithi130.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-10T03:35:05.575 INFO:teuthology.orchestra.run.smithi130.stdout: osd.2 (root=default,host=smithi130) is down 2024-09-10T03:35:05.575 INFO:teuthology.orchestra.run.smithi130.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 158/1668 objects degraded (9.472%), 19 pgs degraded 2024-09-10T03:35:05.575 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-09-10T03:35:05.575 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-09-10T03:35:05.575 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-09-10T03:35:05.576 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-09-10T03:35:05.576 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-09-10T03:35:05.576 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-09-10T03:35:05.576 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-09-10T03:35:05.576 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-09-10T03:35:05.576 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-09-10T03:35:05.576 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-09-10T03:35:05.576 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-09-10T03:35:05.576 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-09-10T03:35:05.576 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-09-10T03:35:05.576 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-09-10T03:35:05.576 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-09-10T03:35:05.576 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-09-10T03:35:05.577 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-09-10T03:35:05.577 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-09-10T03:35:05.577 INFO:teuthology.orchestra.run.smithi130.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-09-10T03:35:05.917 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:05 smithi130 podman[73278]: 2024-09-10 03:35:05.767103547 +0000 UTC m=+0.551912529 container init a871d2ccf96efe15953b07f959ddd4e758b8a8c3c0db3dc5f696512da7851e38 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:35:05.917 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:05 smithi130 podman[73278]: 2024-09-10 03:35:05.77253472 +0000 UTC m=+0.557343694 container start a871d2ccf96efe15953b07f959ddd4e758b8a8c3c0db3dc5f696512da7851e38 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True) 2024-09-10T03:35:05.917 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:05 smithi130 ceph-osd[73298]: -- 172.21.15.130:0/2624878275 <== mon.1 v2:172.21.15.164:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55dac3a361e0 con 0x55dac2bd1c00 2024-09-10T03:35:06.097 INFO:teuthology.orchestra.run.smithi130.stdout: 5 197105 10.76 MB/sec execute 375 sec latency 39.780 ms 2024-09-10T03:35:06.168 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:06 smithi130 ceph-mon[63591]: from='client.34294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:35:06.168 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:06 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/64079991' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:35:06.168 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:06 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:06.168 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:06 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:06.168 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:06 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:35:06.168 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:05 smithi130 bash[73278]: a871d2ccf96efe15953b07f959ddd4e758b8a8c3c0db3dc5f696512da7851e38 2024-09-10T03:35:06.168 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:05 smithi130 systemd[1]: Started Ceph osd.2 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:35:06.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:06 smithi164 ceph-mon[49318]: from='client.34294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:35:06.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:06 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/64079991' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:35:06.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:06 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:06.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:06 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:06.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:06 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:35:07.097 INFO:teuthology.orchestra.run.smithi130.stdout: 5 197751 10.77 MB/sec execute 376 sec latency 32.025 ms 2024-09-10T03:35:07.172 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:07 smithi130 ceph-mon[63591]: pgmap v138: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.1 MiB/s wr, 485 op/s; 168/1683 objects degraded (9.982%) 2024-09-10T03:35:07.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:07 smithi164 ceph-mon[49318]: pgmap v138: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.1 MiB/s wr, 485 op/s; 168/1683 objects degraded (9.982%) 2024-09-10T03:35:07.932 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:07 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2[73291]: 2024-09-10T03:35:07.876+0000 7fc94aaf1640 -1 Falling back to public interface 2024-09-10T03:35:08.097 INFO:teuthology.orchestra.run.smithi130.stdout: 5 198543 10.80 MB/sec execute 377 sec latency 30.672 ms 2024-09-10T03:35:08.195 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:08 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 167/1692 objects degraded (9.870%), 19 pgs degraded (PG_DEGRADED) 2024-09-10T03:35:08.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:08 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 167/1692 objects degraded (9.870%), 19 pgs degraded (PG_DEGRADED) 2024-09-10T03:35:09.097 INFO:teuthology.orchestra.run.smithi130.stdout: 5 199186 10.83 MB/sec execute 378 sec latency 32.504 ms 2024-09-10T03:35:09.205 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:09 smithi130 ceph-mon[63591]: pgmap v139: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.5 MiB/s wr, 438 op/s; 167/1692 objects degraded (9.870%) 2024-09-10T03:35:09.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:09 smithi164 ceph-mon[49318]: pgmap v139: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.5 MiB/s wr, 438 op/s; 167/1692 objects degraded (9.870%) 2024-09-10T03:35:10.097 INFO:teuthology.orchestra.run.smithi130.stdout: 5 199517 10.82 MB/sec execute 379 sec latency 58.499 ms 2024-09-10T03:35:11.097 INFO:teuthology.orchestra.run.smithi130.stdout: 5 199750 10.81 MB/sec execute 380 sec latency 76.866 ms 2024-09-10T03:35:11.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:11 smithi130 ceph-mon[63591]: pgmap v140: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.9 MiB/s wr, 385 op/s; 167/1692 objects degraded (9.870%) 2024-09-10T03:35:11.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:11 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:11.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:11 smithi164 ceph-mon[49318]: pgmap v140: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.9 MiB/s wr, 385 op/s; 167/1692 objects degraded (9.870%) 2024-09-10T03:35:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:11 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:12.097 INFO:teuthology.orchestra.run.smithi130.stdout: 5 200108 10.81 MB/sec execute 381 sec latency 90.737 ms 2024-09-10T03:35:13.097 INFO:teuthology.orchestra.run.smithi130.stdout: 5 200551 10.80 MB/sec execute 382 sec latency 64.732 ms 2024-09-10T03:35:13.281 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:13 smithi130 ceph-mon[63591]: pgmap v141: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.5 MiB/s wr, 717 op/s; 155/1746 objects degraded (8.877%) 2024-09-10T03:35:13.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:13 smithi164 ceph-mon[49318]: pgmap v141: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.5 MiB/s wr, 717 op/s; 155/1746 objects degraded (8.877%) 2024-09-10T03:35:14.097 INFO:teuthology.orchestra.run.smithi130.stdout: 5 200973 10.80 MB/sec execute 383 sec latency 59.096 ms 2024-09-10T03:35:14.541 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:14 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2[73291]: 2024-09-10T03:35:14.487+0000 7fc94aaf1640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-10T03:35:15.097 INFO:teuthology.orchestra.run.smithi130.stdout: 5 201352 10.80 MB/sec execute 384 sec latency 40.938 ms 2024-09-10T03:35:15.170 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:15 smithi130 ceph-mon[63591]: pgmap v142: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.7 MiB/s wr, 539 op/s; 155/1746 objects degraded (8.877%) 2024-09-10T03:35:15.170 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:15 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:15.170 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:15 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:35:15.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:15 smithi164 ceph-mon[49318]: pgmap v142: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.7 MiB/s wr, 539 op/s; 155/1746 objects degraded (8.877%) 2024-09-10T03:35:15.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:15 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:15.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:15 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:35:15.678 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:15 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2[73291]: 2024-09-10T03:35:15.467+0000 7fc94aaf1640 -1 osd.2 98 log_to_monitors true 2024-09-10T03:35:16.097 INFO:teuthology.orchestra.run.smithi130.stdout: 5 201854 10.81 MB/sec execute 385 sec latency 51.671 ms 2024-09-10T03:35:16.185 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:16 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 155/1746 objects degraded (8.877%), 19 pgs degraded (PG_DEGRADED) 2024-09-10T03:35:16.185 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:16 smithi130 ceph-mon[63591]: from='osd.2 [v2:172.21.15.130:6818/366116112,v1:172.21.15.130:6819/366116112]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-10T03:35:16.185 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:16 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:16.185 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:16 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:16.439 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:35:16 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2[73291]: 2024-09-10T03:35:16.243+0000 7fc94289b640 -1 osd.2 98 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-10T03:35:16.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:16 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 155/1746 objects degraded (8.877%), 19 pgs degraded (PG_DEGRADED) 2024-09-10T03:35:16.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:16 smithi164 ceph-mon[49318]: from='osd.2 [v2:172.21.15.130:6818/366116112,v1:172.21.15.130:6819/366116112]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-10T03:35:16.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:16 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:16.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:16 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:17.097 INFO:teuthology.orchestra.run.smithi130.stdout: 5 202237 10.81 MB/sec execute 386 sec latency 41.693 ms 2024-09-10T03:35:17.221 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:17 smithi130 ceph-mon[63591]: pgmap v143: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 778 op/s; 150/1725 objects degraded (8.696%) 2024-09-10T03:35:17.221 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:17 smithi130 ceph-mon[63591]: from='osd.2 [v2:172.21.15.130:6818/366116112,v1:172.21.15.130:6819/366116112]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-10T03:35:17.222 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:17 smithi130 ceph-mon[63591]: osdmap e101: 8 total, 7 up, 8 in 2024-09-10T03:35:17.222 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:17 smithi130 ceph-mon[63591]: from='osd.2 [v2:172.21.15.130:6818/366116112,v1:172.21.15.130:6819/366116112]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-09-10T03:35:17.222 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:17 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:17.222 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:17 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:17.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:17 smithi164 ceph-mon[49318]: pgmap v143: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 778 op/s; 150/1725 objects degraded (8.696%) 2024-09-10T03:35:17.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:17 smithi164 ceph-mon[49318]: from='osd.2 [v2:172.21.15.130:6818/366116112,v1:172.21.15.130:6819/366116112]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-10T03:35:17.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:17 smithi164 ceph-mon[49318]: osdmap e101: 8 total, 7 up, 8 in 2024-09-10T03:35:17.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:17 smithi164 ceph-mon[49318]: from='osd.2 [v2:172.21.15.130:6818/366116112,v1:172.21.15.130:6819/366116112]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-09-10T03:35:17.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:17 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:17.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:17 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:18.097 INFO:teuthology.orchestra.run.smithi130.stdout: 5 202282 10.79 MB/sec execute 387 sec latency 942.215 ms 2024-09-10T03:35:18.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:18 smithi164 ceph-mon[49318]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-10T03:35:18.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:18 smithi164 ceph-mon[49318]: osd.2 [v2:172.21.15.130:6818/366116112,v1:172.21.15.130:6819/366116112] boot 2024-09-10T03:35:18.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:18 smithi164 ceph-mon[49318]: osdmap e102: 8 total, 8 up, 8 in 2024-09-10T03:35:18.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:18 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-10T03:35:18.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:18 smithi130 ceph-mon[63591]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-10T03:35:18.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:18 smithi130 ceph-mon[63591]: osd.2 [v2:172.21.15.130:6818/366116112,v1:172.21.15.130:6819/366116112] boot 2024-09-10T03:35:18.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:18 smithi130 ceph-mon[63591]: osdmap e102: 8 total, 8 up, 8 in 2024-09-10T03:35:18.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:18 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-10T03:35:19.097 INFO:teuthology.orchestra.run.smithi130.stdout: 5 202282 10.76 MB/sec execute 388 sec latency 1942.245 ms 2024-09-10T03:35:19.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:19 smithi130 ceph-mon[63591]: pgmap v146: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 871 op/s; 147/1707 objects degraded (8.612%) 2024-09-10T03:35:19.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:19 smithi130 ceph-mon[63591]: osdmap e103: 8 total, 8 up, 8 in 2024-09-10T03:35:19.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:19 smithi130 ceph-mon[63591]: osdmap e104: 8 total, 8 up, 8 in 2024-09-10T03:35:19.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:19 smithi164 ceph-mon[49318]: pgmap v146: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 871 op/s; 147/1707 objects degraded (8.612%) 2024-09-10T03:35:19.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:19 smithi164 ceph-mon[49318]: osdmap e103: 8 total, 8 up, 8 in 2024-09-10T03:35:19.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:19 smithi164 ceph-mon[49318]: osdmap e104: 8 total, 8 up, 8 in 2024-09-10T03:35:20.097 INFO:teuthology.orchestra.run.smithi130.stdout: 5 202626 10.76 MB/sec execute 389 sec latency 2140.322 ms 2024-09-10T03:35:21.097 INFO:teuthology.orchestra.run.smithi130.stdout: 5 202655 10.73 MB/sec execute 390 sec latency 909.075 ms 2024-09-10T03:35:21.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:21 smithi130 ceph-mon[63591]: pgmap v149: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 303 KiB/s wr, 27 op/s; 147/1707 objects degraded (8.612%) 2024-09-10T03:35:21.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:21 smithi130 ceph-mon[63591]: osdmap e105: 8 total, 8 up, 8 in 2024-09-10T03:35:21.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:21 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 147/1707 objects degraded (8.612%), 19 pgs degraded (PG_DEGRADED) 2024-09-10T03:35:21.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:21 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:21.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:21 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:21.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:21 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:35:21.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:21 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:35:21.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:21 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:21.383 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:21 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:21.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:21 smithi164 ceph-mon[49318]: pgmap v149: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 303 KiB/s wr, 27 op/s; 147/1707 objects degraded (8.612%) 2024-09-10T03:35:21.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:21 smithi164 ceph-mon[49318]: osdmap e105: 8 total, 8 up, 8 in 2024-09-10T03:35:21.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:21 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 147/1707 objects degraded (8.612%), 19 pgs degraded (PG_DEGRADED) 2024-09-10T03:35:21.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:21 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:21.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:21 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:21.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:21 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:35:21.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:21 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:35:21.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:21 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:21.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:21 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:22.097 INFO:teuthology.orchestra.run.smithi130.stdout: 5 203029 10.73 MB/sec execute 391 sec latency 1018.689 ms 2024-09-10T03:35:22.392 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:22 smithi130 ceph-mon[63591]: osdmap e106: 8 total, 8 up, 8 in 2024-09-10T03:35:22.392 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:35:22.392 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:35:22.392 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:35:22.392 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:35:22.392 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-10T03:35:22.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:22 smithi164 ceph-mon[49318]: osdmap e106: 8 total, 8 up, 8 in 2024-09-10T03:35:22.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:35:22.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:35:22.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:35:22.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:35:22.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-10T03:35:23.097 INFO:teuthology.orchestra.run.smithi130.stdout: 5 203303 10.73 MB/sec execute 392 sec latency 853.800 ms 2024-09-10T03:35:23.408 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:23 smithi130 ceph-mon[63591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-10T03:35:23.408 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:23 smithi130 ceph-mon[63591]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-10T03:35:23.408 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:23 smithi130 ceph-mon[63591]: pgmap v152: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 peering, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 366 op/s; 31/1608 objects degraded (1.928%); 1.5 MiB/s, 46 objects/s recovering 2024-09-10T03:35:23.408 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:23 smithi130 ceph-mon[63591]: osdmap e107: 8 total, 8 up, 8 in 2024-09-10T03:35:23.408 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:23 smithi130 ceph-mon[63591]: osdmap e108: 8 total, 8 up, 8 in 2024-09-10T03:35:23.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:23 smithi164 ceph-mon[49318]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-10T03:35:23.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:23 smithi164 ceph-mon[49318]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-10T03:35:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:23 smithi164 ceph-mon[49318]: pgmap v152: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 peering, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 366 op/s; 31/1608 objects degraded (1.928%); 1.5 MiB/s, 46 objects/s recovering 2024-09-10T03:35:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:23 smithi164 ceph-mon[49318]: osdmap e107: 8 total, 8 up, 8 in 2024-09-10T03:35:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:23 smithi164 ceph-mon[49318]: osdmap e108: 8 total, 8 up, 8 in 2024-09-10T03:35:24.100 INFO:teuthology.orchestra.run.smithi130.stdout: 5 203526 10.71 MB/sec execute 393 sec latency 964.835 ms 2024-09-10T03:35:25.100 INFO:teuthology.orchestra.run.smithi130.stdout: 5 203779 10.71 MB/sec execute 394 sec latency 48.647 ms 2024-09-10T03:35:25.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:25 smithi130 ceph-mon[63591]: pgmap v155: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 peering, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 366 op/s; 31/1608 objects degraded (1.928%); 1.5 MiB/s, 46 objects/s recovering 2024-09-10T03:35:25.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:25 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 31/1608 objects degraded (1.928%), 2 pgs degraded (PG_DEGRADED) 2024-09-10T03:35:25.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:25 smithi164 ceph-mon[49318]: pgmap v155: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 peering, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 366 op/s; 31/1608 objects degraded (1.928%); 1.5 MiB/s, 46 objects/s recovering 2024-09-10T03:35:25.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:25 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 31/1608 objects degraded (1.928%), 2 pgs degraded (PG_DEGRADED) 2024-09-10T03:35:26.100 INFO:teuthology.orchestra.run.smithi130.stdout: 5 204024 10.69 MB/sec execute 395 sec latency 76.494 ms 2024-09-10T03:35:26.332 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:26 smithi130 ceph-mon[63591]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/1608 objects degraded (1.928%), 2 pgs degraded) 2024-09-10T03:35:26.332 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:26 smithi130 ceph-mon[63591]: Cluster is now healthy 2024-09-10T03:35:26.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:26 smithi164 ceph-mon[49318]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/1608 objects degraded (1.928%), 2 pgs degraded) 2024-09-10T03:35:26.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:26 smithi164 ceph-mon[49318]: Cluster is now healthy 2024-09-10T03:35:27.101 INFO:teuthology.orchestra.run.smithi130.stdout: 5 204312 10.68 MB/sec execute 396 sec latency 48.683 ms 2024-09-10T03:35:27.373 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:27 smithi130 ceph-mon[63591]: pgmap v156: 97 pgs: 1 peering, 96 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 6.1 MiB/s wr, 593 op/s; 1.0 MiB/s, 40 objects/s recovering 2024-09-10T03:35:27.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:27 smithi164 ceph-mon[49318]: pgmap v156: 97 pgs: 1 peering, 96 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 6.1 MiB/s wr, 593 op/s; 1.0 MiB/s, 40 objects/s recovering 2024-09-10T03:35:28.100 INFO:teuthology.orchestra.run.smithi130.stdout: 5 204533 10.66 MB/sec execute 397 sec latency 74.094 ms 2024-09-10T03:35:29.101 INFO:teuthology.orchestra.run.smithi130.stdout: 5 204880 10.66 MB/sec execute 398 sec latency 40.195 ms 2024-09-10T03:35:29.396 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:29 smithi130 ceph-mon[63591]: pgmap v157: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.1 MiB/s wr, 300 op/s; 140 KiB/s, 15 objects/s recovering 2024-09-10T03:35:29.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:29 smithi164 ceph-mon[49318]: pgmap v157: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.1 MiB/s wr, 300 op/s; 140 KiB/s, 15 objects/s recovering 2024-09-10T03:35:30.109 INFO:teuthology.orchestra.run.smithi130.stdout: 5 205225 10.65 MB/sec execute 399 sec latency 64.988 ms 2024-09-10T03:35:31.109 INFO:teuthology.orchestra.run.smithi130.stdout: 5 205629 10.66 MB/sec execute 400 sec latency 41.940 ms 2024-09-10T03:35:31.159 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:31 smithi130 ceph-mon[63591]: pgmap v158: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.4 MiB/s wr, 239 op/s; 0 B/s, 4 objects/s recovering 2024-09-10T03:35:31.159 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:31 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:31.159 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:31 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:35:31.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:31 smithi164 ceph-mon[49318]: pgmap v158: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.4 MiB/s wr, 239 op/s; 0 B/s, 4 objects/s recovering 2024-09-10T03:35:31.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:31 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:31.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:31 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:35:32.109 INFO:teuthology.orchestra.run.smithi130.stdout: 5 206063 10.67 MB/sec execute 401 sec latency 31.810 ms 2024-09-10T03:35:33.109 INFO:teuthology.orchestra.run.smithi130.stdout: 5 206459 10.67 MB/sec execute 402 sec latency 64.024 ms 2024-09-10T03:35:33.189 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:33 smithi130 ceph-mon[63591]: pgmap v159: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.3 MiB/s wr, 442 op/s; 0 B/s, 3 objects/s recovering 2024-09-10T03:35:33.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:33 smithi164 ceph-mon[49318]: pgmap v159: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.3 MiB/s wr, 442 op/s; 0 B/s, 3 objects/s recovering 2024-09-10T03:35:34.109 INFO:teuthology.orchestra.run.smithi130.stdout: 5 206807 10.67 MB/sec execute 403 sec latency 34.584 ms 2024-09-10T03:35:35.109 INFO:teuthology.orchestra.run.smithi130.stdout: 5 207148 10.66 MB/sec execute 404 sec latency 53.794 ms 2024-09-10T03:35:35.246 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:35 smithi130 ceph-mon[63591]: pgmap v160: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.9 MiB/s wr, 400 op/s; 0 B/s, 3 objects/s recovering 2024-09-10T03:35:35.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:35 smithi164 ceph-mon[49318]: pgmap v160: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.9 MiB/s wr, 400 op/s; 0 B/s, 3 objects/s recovering 2024-09-10T03:35:36.110 INFO:teuthology.orchestra.run.smithi130.stdout: 5 207500 10.66 MB/sec execute 405 sec latency 47.478 ms 2024-09-10T03:35:36.153 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:35:36.285 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:36 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:36.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:36 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:37.110 INFO:teuthology.orchestra.run.smithi130.stdout: 5 207884 10.65 MB/sec execute 406 sec latency 47.507 ms 2024-09-10T03:35:37.198 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:35:37.198 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (6m) 21s ago 15m 19.2M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:35:37.198 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (2m) 21s ago 15m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 8c7cfb7e87a1 2024-09-10T03:35:37.198 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (2m) 2m ago 14m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 243c094648b8 2024-09-10T03:35:37.199 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (6m) 21s ago 15m 56.3M - dad864ee21e9 72ba01ac1871 2024-09-10T03:35:37.199 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (9m) 21s ago 9m 37.2M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:35:37.199 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (9m) 2m ago 9m 15.9M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:35:37.199 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (5m) 21s ago 16m 517M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:35:37.199 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283,8765 running (4m) 2m ago 14m 453M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e4034e8e2a6d 2024-09-10T03:35:37.199 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (4m) 21s ago 16m 51.8M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 22e5cf56527a 2024-09-10T03:35:37.199 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (3m) 2m ago 14m 36.8M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 7d80bd0d15b9 2024-09-10T03:35:37.199 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (8m) 21s ago 8m 273M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:35:37.199 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (8m) 2m ago 8m 87.5M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:35:37.199 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (6m) 21s ago 15m 10.3M - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:35:37.199 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (5m) 2m ago 14m 10.6M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:35:37.199 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (2m) 21s ago 13m 312M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b de3cbff0098b 2024-09-10T03:35:37.199 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (89s) 21s ago 13m 236M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f0722a6d1c90 2024-09-10T03:35:37.200 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (31s) 21s ago 12m 14.4M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b a871d2ccf96e 2024-09-10T03:35:37.200 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (12m) 21s ago 12m 362M 1936M 17.2.0 e1d6a67b021e b32084e1a404 2024-09-10T03:35:37.200 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (12m) 2m ago 12m 504M 2704M 17.2.0 e1d6a67b021e 3b5c9f00f64d 2024-09-10T03:35:37.200 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (11m) 2m ago 11m 559M 2704M 17.2.0 e1d6a67b021e f16357ac331e 2024-09-10T03:35:37.200 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (11m) 2m ago 11m 490M 2704M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:35:37.200 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (10m) 2m ago 10m 396M 2704M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:35:37.200 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (4m) 21s ago 14m 54.5M - 2.51.0 1d3b7f56885b f59b02fac5b3 2024-09-10T03:35:37.302 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:37 smithi130 ceph-mon[63591]: pgmap v161: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.8 MiB/s wr, 576 op/s; 0 B/s, 2 objects/s recovering 2024-09-10T03:35:37.302 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:37 smithi130 ceph-mon[63591]: from='client.44231 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:35:37.302 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-10T03:35:37.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:37 smithi164 ceph-mon[49318]: pgmap v161: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.8 MiB/s wr, 576 op/s; 0 B/s, 2 objects/s recovering 2024-09-10T03:35:37.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:37 smithi164 ceph-mon[49318]: from='client.44231 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:35:37.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-10T03:35:37.765 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:35:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:35:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:35:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:35:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:35:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:35:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:35:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:35:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-10T03:35:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 3 2024-09-10T03:35:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:35:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:35:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:35:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:35:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:35:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-10T03:35:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 7 2024-09-10T03:35:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:35:37.766 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:35:38.110 INFO:teuthology.orchestra.run.smithi130.stdout: 5 208093 10.63 MB/sec execute 407 sec latency 81.020 ms 2024-09-10T03:35:38.271 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:35:38.271 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:35:38.271 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:35:38.271 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:35:38.271 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-09-10T03:35:38.271 INFO:teuthology.orchestra.run.smithi130.stdout: "mon", 2024-09-10T03:35:38.271 INFO:teuthology.orchestra.run.smithi130.stdout: "crash", 2024-09-10T03:35:38.271 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-09-10T03:35:38.271 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-09-10T03:35:38.272 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "9/23 daemons upgraded", 2024-09-10T03:35:38.272 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading osd daemons", 2024-09-10T03:35:38.272 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:35:38.272 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:35:38.325 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:38 smithi130 ceph-mon[63591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-10T03:35:38.325 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:38 smithi130 ceph-mon[63591]: Upgrade: osd.3 is safe to restart 2024-09-10T03:35:38.325 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:38 smithi130 ceph-mon[63591]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:35:38.325 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:38 smithi130 ceph-mon[63591]: from='client.44239 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:35:38.325 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:38 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/1988225220' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:35:38.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:38 smithi164 ceph-mon[49318]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-10T03:35:38.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:38 smithi164 ceph-mon[49318]: Upgrade: osd.3 is safe to restart 2024-09-10T03:35:38.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:38 smithi164 ceph-mon[49318]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:35:38.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:38 smithi164 ceph-mon[49318]: from='client.44239 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:35:38.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:38 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/1988225220' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:35:38.849 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-09-10T03:35:39.110 INFO:teuthology.orchestra.run.smithi130.stdout: 5 208425 10.62 MB/sec execute 408 sec latency 57.761 ms 2024-09-10T03:35:39.339 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:39 smithi130 ceph-mon[63591]: pgmap v162: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.4 MiB/s wr, 431 op/s 2024-09-10T03:35:39.339 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:39 smithi130 ceph-mon[63591]: from='client.44247 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:35:39.339 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:39 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/1569575930' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:35:39.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:39 smithi164 ceph-mon[49318]: pgmap v162: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.4 MiB/s wr, 431 op/s 2024-09-10T03:35:39.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:39 smithi164 ceph-mon[49318]: from='client.44247 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:35:39.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:39 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/1569575930' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:35:40.110 INFO:teuthology.orchestra.run.smithi130.stdout: 5 208881 10.63 MB/sec execute 409 sec latency 40.829 ms 2024-09-10T03:35:40.637 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:40 smithi130 ceph-mon[63591]: Upgrade: Updating osd.3 2024-09-10T03:35:40.637 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:40 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:40.638 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:40 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-10T03:35:40.638 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:40 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:35:40.638 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:40 smithi130 ceph-mon[63591]: Deploying daemon osd.3 on smithi130 2024-09-10T03:35:40.638 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:40 smithi130 ceph-mon[63591]: pgmap v163: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.3 MiB/s wr, 425 op/s 2024-09-10T03:35:40.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:40 smithi164 ceph-mon[49318]: Upgrade: Updating osd.3 2024-09-10T03:35:40.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:40 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:40.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:40 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-10T03:35:40.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:40 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:35:40.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:40 smithi164 ceph-mon[49318]: Deploying daemon osd.3 on smithi130 2024-09-10T03:35:40.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:40 smithi164 ceph-mon[49318]: pgmap v163: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.3 MiB/s wr, 425 op/s 2024-09-10T03:35:41.110 INFO:teuthology.orchestra.run.smithi130.stdout: 5 209272 10.65 MB/sec execute 410 sec latency 36.386 ms 2024-09-10T03:35:42.110 INFO:teuthology.orchestra.run.smithi130.stdout: 5 209612 10.64 MB/sec execute 411 sec latency 51.403 ms 2024-09-10T03:35:42.911 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:42 smithi130 ceph-mon[63591]: pgmap v164: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.4 MiB/s wr, 637 op/s 2024-09-10T03:35:43.110 INFO:teuthology.orchestra.run.smithi130.stdout: 5 209928 10.64 MB/sec execute 412 sec latency 43.372 ms 2024-09-10T03:35:43.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:42 smithi164 ceph-mon[49318]: pgmap v164: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.4 MiB/s wr, 637 op/s 2024-09-10T03:35:44.110 INFO:teuthology.orchestra.run.smithi130.stdout: 5 210271 10.64 MB/sec execute 413 sec latency 86.505 ms 2024-09-10T03:35:44.994 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:44 smithi130 ceph-mon[63591]: pgmap v165: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.6 MiB/s wr, 445 op/s 2024-09-10T03:35:45.110 INFO:teuthology.orchestra.run.smithi130.stdout: 5 210561 10.63 MB/sec execute 414 sec latency 43.328 ms 2024-09-10T03:35:45.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:44 smithi164 ceph-mon[49318]: pgmap v165: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.6 MiB/s wr, 445 op/s 2024-09-10T03:35:45.763 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:45 smithi130 systemd[1]: Stopping Ceph osd.3 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:35:46.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:45 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:35:46.013 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:45 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3[44936]: 2024-09-10T03:35:45.924+0000 7f17a1738700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:35:46.013 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:45 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3[44936]: 2024-09-10T03:35:45.924+0000 7f17a1738700 -1 osd.3 108 *** Got signal Terminated *** 2024-09-10T03:35:46.013 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:45 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3[44936]: 2024-09-10T03:35:45.924+0000 7f17a1738700 -1 osd.3 108 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-10T03:35:46.111 INFO:teuthology.orchestra.run.smithi130.stdout: 5 210901 10.63 MB/sec execute 415 sec latency 79.797 ms 2024-09-10T03:35:46.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:45 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:35:47.110 INFO:teuthology.orchestra.run.smithi130.stdout: 5 211164 10.61 MB/sec execute 416 sec latency 284.919 ms 2024-09-10T03:35:47.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:46 smithi164 ceph-mon[49318]: pgmap v166: 97 pgs: 97 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.3 MiB/s wr, 575 op/s 2024-09-10T03:35:47.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:46 smithi164 ceph-mon[49318]: osd.3 marked itself down and dead 2024-09-10T03:35:47.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:46 smithi130 ceph-mon[63591]: pgmap v166: 97 pgs: 97 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.3 MiB/s wr, 575 op/s 2024-09-10T03:35:47.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:46 smithi130 ceph-mon[63591]: osd.3 marked itself down and dead 2024-09-10T03:35:47.747 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:47 smithi130 podman[75190]: 2024-09-10 03:35:47.487170658 +0000 UTC m=+1.707107719 container died b32084e1a404d6d9d7a63c3bff18605fd21dc0d43b1687d1dcb5b7a39ebbec04 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, vcs-type=git, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, GIT_CLEAN=True, name=centos-stream, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, RELEASE=HEAD, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., ceph=True, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux ) 2024-09-10T03:35:48.007 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:47 smithi130 ceph-mon[63591]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-10T03:35:48.007 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:47 smithi130 ceph-mon[63591]: osdmap e109: 8 total, 7 up, 8 in 2024-09-10T03:35:48.007 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:47 smithi130 podman[75190]: 2024-09-10 03:35:47.963239179 +0000 UTC m=+2.183176239 container cleanup b32084e1a404d6d9d7a63c3bff18605fd21dc0d43b1687d1dcb5b7a39ebbec04 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.buildah.version=1.19.8, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, version=8, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, release=754, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container) 2024-09-10T03:35:48.007 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:47 smithi130 bash[75190]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3 2024-09-10T03:35:48.111 INFO:teuthology.orchestra.run.smithi130.stdout: 5 211262 10.59 MB/sec execute 417 sec latency 1047.734 ms 2024-09-10T03:35:48.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:47 smithi164 ceph-mon[49318]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-10T03:35:48.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:47 smithi164 ceph-mon[49318]: osdmap e109: 8 total, 7 up, 8 in 2024-09-10T03:35:48.481 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:48 smithi130 podman[75205]: 2024-09-10 03:35:48.459824769 +0000 UTC m=+0.975767005 container remove b32084e1a404d6d9d7a63c3bff18605fd21dc0d43b1687d1dcb5b7a39ebbec04 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3, vcs-type=git, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, ceph=True, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, name=centos-stream, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, io.buildah.version=1.19.8) 2024-09-10T03:35:49.023 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:49 smithi130 podman[75292]: 2024-09-10 03:35:48.908529148 +0000 UTC m=+0.024899977 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:35:49.023 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:48 smithi130 ceph-mon[63591]: pgmap v168: 97 pgs: 16 stale+active+clean, 81 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.2 MiB/s wr, 513 op/s 2024-09-10T03:35:49.023 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:48 smithi130 ceph-mon[63591]: osdmap e110: 8 total, 7 up, 8 in 2024-09-10T03:35:49.111 INFO:teuthology.orchestra.run.smithi130.stdout: 5 211688 10.58 MB/sec execute 418 sec latency 32.537 ms 2024-09-10T03:35:49.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:48 smithi164 ceph-mon[49318]: pgmap v168: 97 pgs: 16 stale+active+clean, 81 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.2 MiB/s wr, 513 op/s 2024-09-10T03:35:49.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:48 smithi164 ceph-mon[49318]: osdmap e110: 8 total, 7 up, 8 in 2024-09-10T03:35:49.288 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:49 smithi130 podman[75292]: 2024-09-10 03:35:49.042204666 +0000 UTC m=+0.158575473 container create c21d1aebb98b832af29df1ecd9cfecc74a2d293c76bb748f67a231996ad5e93c (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-10T03:35:49.549 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:49 smithi130 podman[75292]: 2024-09-10 03:35:49.513629412 +0000 UTC m=+0.630000242 container init c21d1aebb98b832af29df1ecd9cfecc74a2d293c76bb748f67a231996ad5e93c (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-10T03:35:49.549 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:49 smithi130 podman[75292]: 2024-09-10 03:35:49.520682345 +0000 UTC m=+0.637053156 container start c21d1aebb98b832af29df1ecd9cfecc74a2d293c76bb748f67a231996ad5e93c (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-deactivate, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:35:49.800 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:49 smithi130 podman[75292]: 2024-09-10 03:35:49.652872948 +0000 UTC m=+0.769243766 container attach c21d1aebb98b832af29df1ecd9cfecc74a2d293c76bb748f67a231996ad5e93c (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-10T03:35:50.067 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:49 smithi130 podman[75292]: 2024-09-10 03:35:49.841458414 +0000 UTC m=+0.957829245 container died c21d1aebb98b832af29df1ecd9cfecc74a2d293c76bb748f67a231996ad5e93c (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1) 2024-09-10T03:35:50.111 INFO:teuthology.orchestra.run.smithi130.stdout: 5 212149 10.59 MB/sec execute 419 sec latency 41.110 ms 2024-09-10T03:35:51.078 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:50 smithi130 ceph-mon[63591]: pgmap v170: 97 pgs: 16 stale+active+clean, 81 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.3 MiB/s wr, 323 op/s 2024-09-10T03:35:51.078 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:50 smithi130 podman[75314]: 2024-09-10 03:35:50.866078195 +0000 UTC m=+1.012534504 container remove c21d1aebb98b832af29df1ecd9cfecc74a2d293c76bb748f67a231996ad5e93c (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-10T03:35:51.078 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:50 smithi130 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-10T03:35:51.078 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:50 smithi130 systemd[1]: Stopped Ceph osd.3 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:35:51.078 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:50 smithi130 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.3.service: Consumed 4min 2.658s CPU time. 2024-09-10T03:35:51.111 INFO:teuthology.orchestra.run.smithi130.stdout: 5 212496 10.58 MB/sec execute 420 sec latency 47.958 ms 2024-09-10T03:35:51.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:50 smithi164 ceph-mon[49318]: pgmap v170: 97 pgs: 16 stale+active+clean, 81 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.3 MiB/s wr, 323 op/s 2024-09-10T03:35:51.335 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:51 smithi130 systemd[1]: Starting Ceph osd.3 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:35:51.591 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:51 smithi130 podman[75411]: 2024-09-10 03:35:51.396214802 +0000 UTC m=+0.025689373 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:35:51.591 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:51 smithi130 podman[75411]: 2024-09-10 03:35:51.539696372 +0000 UTC m=+0.169170942 container create 59decef4f5edd5618547338d0edf8d77f0c91f4f92d1876fcf2b1ce60e53f789 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-activate, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-10T03:35:52.020 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:51 smithi130 ceph-mon[63591]: Health check failed: Degraded data redundancy: 216/1719 objects degraded (12.565%), 27 pgs degraded (PG_DEGRADED) 2024-09-10T03:35:52.111 INFO:teuthology.orchestra.run.smithi130.stdout: 5 212849 10.60 MB/sec execute 421 sec latency 52.292 ms 2024-09-10T03:35:52.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:51 smithi164 ceph-mon[49318]: Health check failed: Degraded data redundancy: 216/1719 objects degraded (12.565%), 27 pgs degraded (PG_DEGRADED) 2024-09-10T03:35:52.299 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:52 smithi130 podman[75411]: 2024-09-10 03:35:52.063574102 +0000 UTC m=+0.693048672 container init 59decef4f5edd5618547338d0edf8d77f0c91f4f92d1876fcf2b1ce60e53f789 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:35:52.299 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:52 smithi130 podman[75411]: 2024-09-10 03:35:52.069049747 +0000 UTC m=+0.698524321 container start 59decef4f5edd5618547338d0edf8d77f0c91f4f92d1876fcf2b1ce60e53f789 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-activate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-10T03:35:52.299 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:52 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-activate[75426]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:35:52.299 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:52 smithi130 podman[75411]: 2024-09-10 03:35:52.249368498 +0000 UTC m=+0.878843069 container attach 59decef4f5edd5618547338d0edf8d77f0c91f4f92d1876fcf2b1ce60e53f789 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-10T03:35:52.300 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:52 smithi130 bash[75411]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:35:52.300 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:52 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-activate[75426]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:35:52.300 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:52 smithi130 bash[75411]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:35:53.058 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:52 smithi130 ceph-mon[63591]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 5.9 MiB/s wr, 602 op/s; 216/1719 objects degraded (12.565%) 2024-09-10T03:35:53.058 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:52 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-activate[75426]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-10T03:35:53.058 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:52 smithi130 bash[75411]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-10T03:35:53.058 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:52 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-activate[75426]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-10T03:35:53.058 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:52 smithi130 bash[75411]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-10T03:35:53.111 INFO:teuthology.orchestra.run.smithi130.stdout: 5 213220 10.60 MB/sec execute 422 sec latency 51.804 ms 2024-09-10T03:35:53.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:52 smithi164 ceph-mon[49318]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 5.9 MiB/s wr, 602 op/s; 216/1719 objects degraded (12.565%) 2024-09-10T03:35:53.312 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:53 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-activate[75426]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-10T03:35:53.312 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:53 smithi130 bash[75411]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-10T03:35:53.312 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:53 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-activate[75426]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-10T03:35:53.313 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:53 smithi130 bash[75411]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-10T03:35:53.313 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:53 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-activate[75426]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-10T03:35:53.313 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:53 smithi130 bash[75411]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-10T03:35:53.313 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:53 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-activate[75426]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-10T03:35:53.313 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:53 smithi130 bash[75411]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-10T03:35:53.313 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:53 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-activate[75426]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-10T03:35:53.313 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:53 smithi130 bash[75411]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-10T03:35:53.313 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:53 smithi130 podman[75411]: 2024-09-10 03:35:53.281769318 +0000 UTC m=+1.911243899 container died 59decef4f5edd5618547338d0edf8d77f0c91f4f92d1876fcf2b1ce60e53f789 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-activate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1) 2024-09-10T03:35:54.111 INFO:teuthology.orchestra.run.smithi130.stdout: 5 213500 10.59 MB/sec execute 423 sec latency 49.291 ms 2024-09-10T03:35:54.617 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:54 smithi130 podman[75570]: 2024-09-10 03:35:54.51874091 +0000 UTC m=+1.224983441 container remove 59decef4f5edd5618547338d0edf8d77f0c91f4f92d1876fcf2b1ce60e53f789 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-activate, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-10T03:35:54.883 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:54 smithi130 podman[75628]: 2024-09-10 03:35:54.655501347 +0000 UTC m=+0.019692760 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:35:54.883 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:54 smithi130 podman[75628]: 2024-09-10 03:35:54.782509167 +0000 UTC m=+0.146700576 container create f2bc83e91fa2274e1f52f6e6de0e3731522db9d8237a3709c0e21014e3736cfe (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-09-10T03:35:55.117 INFO:teuthology.orchestra.run.smithi130.stdout: 5 213845 10.59 MB/sec execute 424 sec latency 46.354 ms 2024-09-10T03:35:55.148 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:54 smithi130 ceph-mon[63591]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s wr, 407 op/s; 216/1719 objects degraded (12.565%) 2024-09-10T03:35:55.148 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:55 smithi130 podman[75628]: 2024-09-10 03:35:55.060553693 +0000 UTC m=+0.424745117 container init f2bc83e91fa2274e1f52f6e6de0e3731522db9d8237a3709c0e21014e3736cfe (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=) 2024-09-10T03:35:55.148 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:55 smithi130 podman[75628]: 2024-09-10 03:35:55.072710851 +0000 UTC m=+0.436902267 container start f2bc83e91fa2274e1f52f6e6de0e3731522db9d8237a3709c0e21014e3736cfe (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-10T03:35:55.148 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:55 smithi130 ceph-osd[75647]: -- 172.21.15.130:0/4263410911 <== mon.0 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55eeb320c1e0 con 0x55eeb319c000 2024-09-10T03:35:55.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:54 smithi164 ceph-mon[49318]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s wr, 407 op/s; 216/1719 objects degraded (12.565%) 2024-09-10T03:35:55.403 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:55 smithi130 bash[75628]: f2bc83e91fa2274e1f52f6e6de0e3731522db9d8237a3709c0e21014e3736cfe 2024-09-10T03:35:55.403 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:55 smithi130 systemd[1]: Started Ceph osd.3 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:35:56.117 INFO:teuthology.orchestra.run.smithi130.stdout: 5 214158 10.58 MB/sec execute 425 sec latency 43.331 ms 2024-09-10T03:35:56.440 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:56 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:56.440 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:56 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:56.440 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:56 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:35:56.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:56 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:56.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:56 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:35:56.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:56 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:35:57.117 INFO:teuthology.orchestra.run.smithi130.stdout: 5 214514 10.58 MB/sec execute 426 sec latency 53.552 ms 2024-09-10T03:35:57.455 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:57 smithi130 ceph-mon[63591]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.9 MiB/s wr, 501 op/s; 217/1722 objects degraded (12.602%) 2024-09-10T03:35:57.455 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:35:57 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3[75643]: 2024-09-10T03:35:57.240+0000 7fcf57492640 -1 Falling back to public interface 2024-09-10T03:35:57.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:57 smithi164 ceph-mon[49318]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.9 MiB/s wr, 501 op/s; 217/1722 objects degraded (12.602%) 2024-09-10T03:35:58.117 INFO:teuthology.orchestra.run.smithi130.stdout: 5 214802 10.56 MB/sec execute 427 sec latency 101.179 ms 2024-09-10T03:35:58.484 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:58 smithi130 ceph-mon[63591]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.7 MiB/s wr, 463 op/s; 220/1740 objects degraded (12.644%) 2024-09-10T03:35:58.484 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:35:58 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 220/1740 objects degraded (12.644%), 27 pgs degraded (PG_DEGRADED) 2024-09-10T03:35:58.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:58 smithi164 ceph-mon[49318]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.7 MiB/s wr, 463 op/s; 220/1740 objects degraded (12.644%) 2024-09-10T03:35:58.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:35:58 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 220/1740 objects degraded (12.644%), 27 pgs degraded (PG_DEGRADED) 2024-09-10T03:35:59.117 INFO:teuthology.orchestra.run.smithi130.stdout: 5 215124 10.55 MB/sec execute 428 sec latency 76.512 ms 2024-09-10T03:36:00.117 INFO:teuthology.orchestra.run.smithi130.stdout: 5 215438 10.54 MB/sec execute 429 sec latency 97.295 ms 2024-09-10T03:36:01.063 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:01 smithi130 ceph-mon[63591]: pgmap v175: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.9 MiB/s wr, 388 op/s; 220/1740 objects degraded (12.644%) 2024-09-10T03:36:01.063 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:01.063 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:36:01.117 INFO:teuthology.orchestra.run.smithi130.stdout: 5 215707 10.53 MB/sec execute 430 sec latency 61.079 ms 2024-09-10T03:36:01.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:01 smithi164 ceph-mon[49318]: pgmap v175: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.9 MiB/s wr, 388 op/s; 220/1740 objects degraded (12.644%) 2024-09-10T03:36:01.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:36:02.117 INFO:teuthology.orchestra.run.smithi130.stdout: 5 215928 10.53 MB/sec execute 431 sec latency 114.894 ms 2024-09-10T03:36:03.117 INFO:teuthology.orchestra.run.smithi130.stdout: 5 216195 10.52 MB/sec execute 432 sec latency 93.339 ms 2024-09-10T03:36:03.134 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:03 smithi130 ceph-mon[63591]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.6 MiB/s wr, 584 op/s; 211/1638 objects degraded (12.882%) 2024-09-10T03:36:03.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:03 smithi164 ceph-mon[49318]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.6 MiB/s wr, 584 op/s; 211/1638 objects degraded (12.882%) 2024-09-10T03:36:04.117 INFO:teuthology.orchestra.run.smithi130.stdout: 5 216468 10.53 MB/sec execute 433 sec latency 53.945 ms 2024-09-10T03:36:04.968 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:04 smithi130 ceph-mon[63591]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 397 op/s; 211/1638 objects degraded (12.882%) 2024-09-10T03:36:04.968 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:04 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:04.968 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:04 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:04.968 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:04 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:04.968 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:04 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:05.117 INFO:teuthology.orchestra.run.smithi130.stdout: 5 216730 10.52 MB/sec execute 434 sec latency 118.880 ms 2024-09-10T03:36:05.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:04 smithi164 ceph-mon[49318]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 397 op/s; 211/1638 objects degraded (12.882%) 2024-09-10T03:36:05.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:04 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:05.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:04 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:05.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:04 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:05.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:04 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:05.232 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:36:05 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3[75643]: 2024-09-10T03:36:05.095+0000 7fcf57492640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-10T03:36:05.997 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:05 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 211/1638 objects degraded (12.882%), 27 pgs degraded (PG_DEGRADED) 2024-09-10T03:36:06.117 INFO:teuthology.orchestra.run.smithi130.stdout: 5 216959 10.51 MB/sec execute 435 sec latency 93.922 ms 2024-09-10T03:36:06.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:05 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 211/1638 objects degraded (12.882%), 27 pgs degraded (PG_DEGRADED) 2024-09-10T03:36:06.515 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:36:06 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3[75643]: 2024-09-10T03:36:06.309+0000 7fcf57492640 -1 osd.3 108 log_to_monitors true 2024-09-10T03:36:07.021 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:06 smithi130 ceph-mon[63591]: pgmap v178: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.4 MiB/s wr, 533 op/s; 205/1662 objects degraded (12.335%) 2024-09-10T03:36:07.021 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:06 smithi130 ceph-mon[63591]: from='osd.3 [v2:172.21.15.130:6826/1598225548,v1:172.21.15.130:6827/1598225548]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-10T03:36:07.021 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:06 smithi130 ceph-mon[63591]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-10T03:36:07.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 217197 10.50 MB/sec execute 436 sec latency 79.775 ms 2024-09-10T03:36:07.281 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:36:07 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3[75643]: 2024-09-10T03:36:07.037+0000 7fcf4f23c640 -1 osd.3 108 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-10T03:36:07.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:06 smithi164 ceph-mon[49318]: pgmap v178: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.4 MiB/s wr, 533 op/s; 205/1662 objects degraded (12.335%) 2024-09-10T03:36:07.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:06 smithi164 ceph-mon[49318]: from='osd.3 [v2:172.21.15.130:6826/1598225548,v1:172.21.15.130:6827/1598225548]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-10T03:36:07.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:06 smithi164 ceph-mon[49318]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-10T03:36:08.055 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:07 smithi130 ceph-mon[63591]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-10T03:36:08.055 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:07 smithi130 ceph-mon[63591]: osdmap e111: 8 total, 7 up, 8 in 2024-09-10T03:36:08.055 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:07 smithi130 ceph-mon[63591]: from='osd.3 [v2:172.21.15.130:6826/1598225548,v1:172.21.15.130:6827/1598225548]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-09-10T03:36:08.055 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:07 smithi130 ceph-mon[63591]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-09-10T03:36:08.117 INFO:teuthology.orchestra.run.smithi130.stdout: 5 217439 10.49 MB/sec execute 437 sec latency 97.739 ms 2024-09-10T03:36:08.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:07 smithi164 ceph-mon[49318]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-10T03:36:08.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:07 smithi164 ceph-mon[49318]: osdmap e111: 8 total, 7 up, 8 in 2024-09-10T03:36:08.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:07 smithi164 ceph-mon[49318]: from='osd.3 [v2:172.21.15.130:6826/1598225548,v1:172.21.15.130:6827/1598225548]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-09-10T03:36:08.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:07 smithi164 ceph-mon[49318]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-09-10T03:36:09.073 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:08 smithi130 ceph-mon[63591]: pgmap v180: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.0 MiB/s wr, 408 op/s; 203/1647 objects degraded (12.325%) 2024-09-10T03:36:09.073 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:08 smithi130 ceph-mon[63591]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-10T03:36:09.073 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:08 smithi130 ceph-mon[63591]: osd.3 [v2:172.21.15.130:6826/1598225548,v1:172.21.15.130:6827/1598225548] boot 2024-09-10T03:36:09.073 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:08 smithi130 ceph-mon[63591]: osdmap e112: 8 total, 8 up, 8 in 2024-09-10T03:36:09.073 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:08 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-10T03:36:09.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 217568 10.47 MB/sec execute 438 sec latency 1097.775 ms 2024-09-10T03:36:09.353 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:36:09.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:08 smithi164 ceph-mon[49318]: pgmap v180: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.0 MiB/s wr, 408 op/s; 203/1647 objects degraded (12.325%) 2024-09-10T03:36:09.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:08 smithi164 ceph-mon[49318]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-10T03:36:09.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:08 smithi164 ceph-mon[49318]: osd.3 [v2:172.21.15.130:6826/1598225548,v1:172.21.15.130:6827/1598225548] boot 2024-09-10T03:36:09.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:08 smithi164 ceph-mon[49318]: osdmap e112: 8 total, 8 up, 8 in 2024-09-10T03:36:09.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:08 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-10T03:36:10.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 217583 10.45 MB/sec execute 439 sec latency 2054.109 ms 2024-09-10T03:36:10.141 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:36:10.142 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (6m) 6s ago 15m 19.4M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:36:10.142 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (3m) 6s ago 15m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 8c7cfb7e87a1 2024-09-10T03:36:10.142 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (3m) 3m ago 15m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 243c094648b8 2024-09-10T03:36:10.142 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (6m) 6s ago 15m 55.9M - dad864ee21e9 72ba01ac1871 2024-09-10T03:36:10.142 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (9m) 6s ago 9m 37.4M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:36:10.142 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (9m) 3m ago 9m 15.9M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:36:10.142 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (6m) 6s ago 17m 519M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:36:10.142 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283,8765 running (5m) 3m ago 15m 453M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e4034e8e2a6d 2024-09-10T03:36:10.142 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (4m) 6s ago 17m 53.2M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 22e5cf56527a 2024-09-10T03:36:10.142 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (4m) 3m ago 15m 36.8M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 7d80bd0d15b9 2024-09-10T03:36:10.142 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (9m) 6s ago 9m 284M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:36:10.142 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (9m) 3m ago 9m 87.5M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:36:10.142 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (6m) 6s ago 15m 10.7M - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:36:10.143 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (6m) 3m ago 15m 10.6M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:36:10.143 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (2m) 6s ago 14m 353M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b de3cbff0098b 2024-09-10T03:36:10.143 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (2m) 6s ago 13m 273M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f0722a6d1c90 2024-09-10T03:36:10.143 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (64s) 6s ago 13m 209M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b a871d2ccf96e 2024-09-10T03:36:10.143 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (15s) 6s ago 12m 14.2M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f2bc83e91fa2 2024-09-10T03:36:10.143 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (12m) 3m ago 12m 504M 2704M 17.2.0 e1d6a67b021e 3b5c9f00f64d 2024-09-10T03:36:10.143 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (12m) 3m ago 12m 559M 2704M 17.2.0 e1d6a67b021e f16357ac331e 2024-09-10T03:36:10.143 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (11m) 3m ago 11m 490M 2704M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:36:10.143 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (11m) 3m ago 11m 396M 2704M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:36:10.143 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (5m) 6s ago 14m 54.5M - 2.51.0 1d3b7f56885b f59b02fac5b3 2024-09-10T03:36:10.242 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:09 smithi130 ceph-mon[63591]: osdmap e113: 8 total, 8 up, 8 in 2024-09-10T03:36:10.242 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:09 smithi130 ceph-mon[63591]: osdmap e114: 8 total, 8 up, 8 in 2024-09-10T03:36:10.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:09 smithi164 ceph-mon[49318]: osdmap e113: 8 total, 8 up, 8 in 2024-09-10T03:36:10.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:09 smithi164 ceph-mon[49318]: osdmap e114: 8 total, 8 up, 8 in 2024-09-10T03:36:10.680 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:36:10.680 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:36:10.680 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:36:10.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:36:10.681 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:36:10.681 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:36:10.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:36:10.681 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:36:10.681 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-10T03:36:10.681 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 4 2024-09-10T03:36:10.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:36:10.681 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:36:10.681 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:36:10.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:36:10.681 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:36:10.681 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-10T03:36:10.682 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 8 2024-09-10T03:36:10.682 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:36:10.682 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:36:11.095 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:36:11.095 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:36:11.095 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:36:11.095 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:36:11.095 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-09-10T03:36:11.095 INFO:teuthology.orchestra.run.smithi130.stdout: "mon", 2024-09-10T03:36:11.095 INFO:teuthology.orchestra.run.smithi130.stdout: "crash", 2024-09-10T03:36:11.095 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-09-10T03:36:11.095 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-09-10T03:36:11.095 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "10/23 daemons upgraded", 2024-09-10T03:36:11.095 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading osd daemons", 2024-09-10T03:36:11.095 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:36:11.096 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:36:11.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 217742 10.43 MB/sec execute 440 sec latency 877.058 ms 2024-09-10T03:36:11.279 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:11 smithi130 ceph-mon[63591]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:36:11.279 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:11 smithi130 ceph-mon[63591]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:36:11.279 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:11 smithi130 ceph-mon[63591]: pgmap v183: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.2 MiB/s wr, 285 op/s; 203/1647 objects degraded (12.325%) 2024-09-10T03:36:11.279 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:11 smithi130 ceph-mon[63591]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:36:11.279 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:11 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:11.279 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:11 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:11.279 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:11 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:36:11.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:11 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:36:11.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:11 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:11.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:11 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:11.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:11 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 203/1647 objects degraded (12.325%), 27 pgs degraded (PG_DEGRADED) 2024-09-10T03:36:11.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:11 smithi130 ceph-mon[63591]: osdmap e115: 8 total, 8 up, 8 in 2024-09-10T03:36:11.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:11 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:36:11.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:11 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:36:11.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:11 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:36:11.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:11 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:36:11.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:11 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-10T03:36:11.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:11 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/627687945' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:36:11.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:11 smithi164 ceph-mon[49318]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:36:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:11 smithi164 ceph-mon[49318]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:36:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:11 smithi164 ceph-mon[49318]: pgmap v183: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.2 MiB/s wr, 285 op/s; 203/1647 objects degraded (12.325%) 2024-09-10T03:36:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:11 smithi164 ceph-mon[49318]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:36:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:11 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:11 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:11 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:36:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:11 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:36:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:11 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:11 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:11 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 203/1647 objects degraded (12.325%), 27 pgs degraded (PG_DEGRADED) 2024-09-10T03:36:11.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:11 smithi164 ceph-mon[49318]: osdmap e115: 8 total, 8 up, 8 in 2024-09-10T03:36:11.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:11 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:36:11.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:11 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:36:11.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:11 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:36:11.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:11 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:36:11.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:11 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-10T03:36:11.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:11 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/627687945' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:36:11.601 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_WARN Degraded data redundancy: 203/1647 objects degraded (12.325%), 27 pgs degraded 2024-09-10T03:36:11.602 INFO:teuthology.orchestra.run.smithi130.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 203/1647 objects degraded (12.325%), 27 pgs degraded 2024-09-10T03:36:11.602 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-09-10T03:36:11.602 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-09-10T03:36:11.602 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.a is active+undersized+degraded, acting [1,7] 2024-09-10T03:36:11.602 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.d is active+undersized+degraded, acting [1,4] 2024-09-10T03:36:11.602 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.e is active+undersized+degraded, acting [2,7] 2024-09-10T03:36:11.602 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.12 is active+undersized+degraded, acting [5,7] 2024-09-10T03:36:11.602 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-09-10T03:36:11.602 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-09-10T03:36:11.602 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-09-10T03:36:11.603 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-09-10T03:36:11.603 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-09-10T03:36:11.603 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-09-10T03:36:11.603 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-09-10T03:36:11.603 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-09-10T03:36:11.603 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-09-10T03:36:11.603 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-09-10T03:36:11.603 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-09-10T03:36:11.603 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.c is active+undersized+degraded, acting [5,6] 2024-09-10T03:36:11.603 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-09-10T03:36:11.603 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-09-10T03:36:11.603 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-09-10T03:36:11.603 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-09-10T03:36:11.603 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-09-10T03:36:11.603 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-09-10T03:36:11.604 INFO:teuthology.orchestra.run.smithi130.stdout: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-09-10T03:36:11.604 INFO:teuthology.orchestra.run.smithi130.stdout: pg 4.14 is active+undersized+degraded, acting [1,7] 2024-09-10T03:36:11.604 INFO:teuthology.orchestra.run.smithi130.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-09-10T03:36:12.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 217980 10.42 MB/sec execute 441 sec latency 992.691 ms 2024-09-10T03:36:12.300 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:12 smithi130 ceph-mon[63591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-10T03:36:12.300 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:12 smithi130 ceph-mon[63591]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-10T03:36:12.300 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:12 smithi130 ceph-mon[63591]: from='client.34350 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:36:12.301 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:12 smithi130 ceph-mon[63591]: osdmap e116: 8 total, 8 up, 8 in 2024-09-10T03:36:12.301 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:12 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/3149346317' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:36:12.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:12 smithi164 ceph-mon[49318]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-10T03:36:12.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:12 smithi164 ceph-mon[49318]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-10T03:36:12.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:12 smithi164 ceph-mon[49318]: from='client.34350 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:36:12.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:12 smithi164 ceph-mon[49318]: osdmap e116: 8 total, 8 up, 8 in 2024-09-10T03:36:12.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:12 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/3149346317' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:36:13.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 218211 10.41 MB/sec execute 442 sec latency 87.790 ms 2024-09-10T03:36:13.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:13 smithi130 ceph-mon[63591]: pgmap v187: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+undersized, 9 active+undersized+degraded, 75 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.9 MiB/s wr, 376 op/s; 224/1551 objects degraded (14.442%); 84 KiB/s, 14 objects/s recovering 2024-09-10T03:36:13.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:13 smithi130 ceph-mon[63591]: osdmap e117: 8 total, 8 up, 8 in 2024-09-10T03:36:13.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:13 smithi164 ceph-mon[49318]: pgmap v187: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 4 active+undersized, 9 active+undersized+degraded, 75 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.9 MiB/s wr, 376 op/s; 224/1551 objects degraded (14.442%); 84 KiB/s, 14 objects/s recovering 2024-09-10T03:36:13.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:13 smithi164 ceph-mon[49318]: osdmap e117: 8 total, 8 up, 8 in 2024-09-10T03:36:14.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 218395 10.40 MB/sec execute 443 sec latency 818.647 ms 2024-09-10T03:36:14.425 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:14 smithi130 ceph-mon[63591]: osdmap e118: 8 total, 8 up, 8 in 2024-09-10T03:36:14.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:14 smithi164 ceph-mon[49318]: osdmap e118: 8 total, 8 up, 8 in 2024-09-10T03:36:15.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 218615 10.39 MB/sec execute 444 sec latency 993.977 ms 2024-09-10T03:36:15.456 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:15 smithi130 ceph-mon[63591]: pgmap v190: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 88 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.0 MiB/s wr, 387 op/s; 179/1548 objects degraded (11.563%); 1.1 MiB/s, 30 objects/s recovering 2024-09-10T03:36:15.456 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:15 smithi130 ceph-mon[63591]: osdmap e119: 8 total, 8 up, 8 in 2024-09-10T03:36:15.456 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:15 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:15.456 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:15 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:36:15.456 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:15 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 179/1548 objects degraded (11.563%), 9 pgs degraded (PG_DEGRADED) 2024-09-10T03:36:15.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:15 smithi164 ceph-mon[49318]: pgmap v190: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 88 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.0 MiB/s wr, 387 op/s; 179/1548 objects degraded (11.563%); 1.1 MiB/s, 30 objects/s recovering 2024-09-10T03:36:15.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:15 smithi164 ceph-mon[49318]: osdmap e119: 8 total, 8 up, 8 in 2024-09-10T03:36:15.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:15 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:15.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:15 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:36:15.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:15 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 179/1548 objects degraded (11.563%), 9 pgs degraded (PG_DEGRADED) 2024-09-10T03:36:16.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 218967 10.38 MB/sec execute 445 sec latency 855.095 ms 2024-09-10T03:36:16.475 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:16 smithi130 ceph-mon[63591]: osdmap e120: 8 total, 8 up, 8 in 2024-09-10T03:36:16.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:16 smithi164 ceph-mon[49318]: osdmap e120: 8 total, 8 up, 8 in 2024-09-10T03:36:17.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 219326 10.37 MB/sec execute 446 sec latency 68.566 ms 2024-09-10T03:36:17.494 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:17 smithi130 ceph-mon[63591]: pgmap v193: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s wr, 345 op/s; 87/1527 objects degraded (5.697%); 1.3 MiB/s, 39 objects/s recovering 2024-09-10T03:36:17.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:17 smithi164 ceph-mon[49318]: pgmap v193: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s wr, 345 op/s; 87/1527 objects degraded (5.697%); 1.3 MiB/s, 39 objects/s recovering 2024-09-10T03:36:18.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 219638 10.37 MB/sec execute 447 sec latency 42.680 ms 2024-09-10T03:36:18.515 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:18 smithi130 ceph-mon[63591]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 87/1527 objects degraded (5.697%), 3 pgs degraded) 2024-09-10T03:36:18.515 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:18 smithi130 ceph-mon[63591]: Cluster is now healthy 2024-09-10T03:36:18.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:18 smithi164 ceph-mon[49318]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 87/1527 objects degraded (5.697%), 3 pgs degraded) 2024-09-10T03:36:18.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:18 smithi164 ceph-mon[49318]: Cluster is now healthy 2024-09-10T03:36:19.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 220067 10.39 MB/sec execute 448 sec latency 64.192 ms 2024-09-10T03:36:19.543 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:19 smithi130 ceph-mon[63591]: pgmap v194: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 2.7 MiB/s wr, 353 op/s; 400 KiB/s, 33 objects/s recovering 2024-09-10T03:36:19.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:19 smithi164 ceph-mon[49318]: pgmap v194: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 2.7 MiB/s wr, 353 op/s; 400 KiB/s, 33 objects/s recovering 2024-09-10T03:36:20.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 220377 10.38 MB/sec execute 449 sec latency 73.334 ms 2024-09-10T03:36:20.580 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:20 smithi130 ceph-mon[63591]: pgmap v195: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.3 MiB/s wr, 300 op/s; 340 KiB/s, 28 objects/s recovering 2024-09-10T03:36:20.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:20 smithi164 ceph-mon[49318]: pgmap v195: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.3 MiB/s wr, 300 op/s; 340 KiB/s, 28 objects/s recovering 2024-09-10T03:36:21.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 220653 10.39 MB/sec execute 450 sec latency 56.235 ms 2024-09-10T03:36:22.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 220944 10.38 MB/sec execute 451 sec latency 49.650 ms 2024-09-10T03:36:22.862 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:22 smithi130 ceph-mon[63591]: pgmap v196: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 494 op/s; 277 KiB/s, 22 objects/s recovering 2024-09-10T03:36:23.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 221242 10.37 MB/sec execute 452 sec latency 73.513 ms 2024-09-10T03:36:23.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:22 smithi164 ceph-mon[49318]: pgmap v196: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 494 op/s; 277 KiB/s, 22 objects/s recovering 2024-09-10T03:36:24.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 221577 10.37 MB/sec execute 453 sec latency 63.497 ms 2024-09-10T03:36:24.946 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:24 smithi130 ceph-mon[63591]: pgmap v197: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.4 MiB/s wr, 429 op/s; 233 KiB/s, 19 objects/s recovering 2024-09-10T03:36:25.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 221873 10.36 MB/sec execute 454 sec latency 60.095 ms 2024-09-10T03:36:25.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:24 smithi164 ceph-mon[49318]: pgmap v197: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.4 MiB/s wr, 429 op/s; 233 KiB/s, 19 objects/s recovering 2024-09-10T03:36:26.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 222181 10.35 MB/sec execute 455 sec latency 65.673 ms 2024-09-10T03:36:26.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:26 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:26.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:26 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-10T03:36:26.285 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:26 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:26.285 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:26 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-10T03:36:27.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 222446 10.33 MB/sec execute 456 sec latency 101.052 ms 2024-09-10T03:36:27.329 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:27 smithi130 ceph-mon[63591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-10T03:36:27.329 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:27 smithi130 ceph-mon[63591]: Upgrade: osd.4 is safe to restart 2024-09-10T03:36:27.329 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:27 smithi130 ceph-mon[63591]: pgmap v198: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 5.3 MiB/s wr, 497 op/s; 211 KiB/s, 17 objects/s recovering 2024-09-10T03:36:27.329 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:27 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:27.329 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:27 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-10T03:36:27.329 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:27 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:36:27.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:27 smithi164 ceph-mon[49318]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-10T03:36:27.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:27 smithi164 ceph-mon[49318]: Upgrade: osd.4 is safe to restart 2024-09-10T03:36:27.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:27 smithi164 ceph-mon[49318]: pgmap v198: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 5.3 MiB/s wr, 497 op/s; 211 KiB/s, 17 objects/s recovering 2024-09-10T03:36:27.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:27 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:27.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:27 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-10T03:36:27.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:27 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:36:28.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 222718 10.32 MB/sec execute 457 sec latency 99.369 ms 2024-09-10T03:36:28.351 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:28 smithi130 ceph-mon[63591]: Upgrade: Updating osd.4 2024-09-10T03:36:28.351 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:28 smithi130 ceph-mon[63591]: Deploying daemon osd.4 on smithi164 2024-09-10T03:36:28.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:28 smithi164 ceph-mon[49318]: Upgrade: Updating osd.4 2024-09-10T03:36:28.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:28 smithi164 ceph-mon[49318]: Deploying daemon osd.4 on smithi164 2024-09-10T03:36:29.121 INFO:teuthology.orchestra.run.smithi130.stdout: 5 222911 10.31 MB/sec execute 458 sec latency 52.695 ms 2024-09-10T03:36:29.388 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:29 smithi130 ceph-mon[63591]: pgmap v199: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.5 MiB/s wr, 432 op/s; 81 KiB/s, 7 objects/s recovering 2024-09-10T03:36:29.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:29 smithi164 ceph-mon[49318]: pgmap v199: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.5 MiB/s wr, 432 op/s; 81 KiB/s, 7 objects/s recovering 2024-09-10T03:36:30.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 223124 10.31 MB/sec execute 459 sec latency 67.653 ms 2024-09-10T03:36:30.458 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:30 smithi164 systemd[1]: Stopping Ceph osd.4 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:36:31.058 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:30 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[29939]: 2024-09-10T03:36:30.784+0000 7f018c4dc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:36:31.058 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:30 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[29939]: 2024-09-10T03:36:30.784+0000 7f018c4dc700 -1 osd.4 120 *** Got signal Terminated *** 2024-09-10T03:36:31.059 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:30 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[29939]: 2024-09-10T03:36:30.784+0000 7f018c4dc700 -1 osd.4 120 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-10T03:36:31.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 223371 10.31 MB/sec execute 460 sec latency 56.024 ms 2024-09-10T03:36:31.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:31 smithi130 ceph-mon[63591]: pgmap v200: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 381 op/s 2024-09-10T03:36:31.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:31 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:31.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:31 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:36:31.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:31 smithi130 ceph-mon[63591]: osd.4 marked itself down and dead 2024-09-10T03:36:31.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:31 smithi164 ceph-mon[49318]: pgmap v200: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 381 op/s 2024-09-10T03:36:31.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:31 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:31.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:31 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:36:31.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:31 smithi164 ceph-mon[49318]: osd.4 marked itself down and dead 2024-09-10T03:36:32.066 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:32 smithi164 ceph-mon[49318]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-10T03:36:32.066 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:32 smithi164 ceph-mon[49318]: osdmap e121: 8 total, 7 up, 8 in 2024-09-10T03:36:32.066 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:31 smithi164 podman[52726]: 2024-09-10 03:36:31.812991069 +0000 UTC m=+1.144023150 container died 3b5c9f00f64dcf30da853c28c3fcfde9568edc12aefa87eae461e6b361a46d4f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, architecture=x86_64, vcs-type=git, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream) 2024-09-10T03:36:32.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 223391 10.29 MB/sec execute 461 sec latency 935.707 ms 2024-09-10T03:36:32.310 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:32 smithi130 ceph-mon[63591]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-10T03:36:32.310 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:32 smithi130 ceph-mon[63591]: osdmap e121: 8 total, 7 up, 8 in 2024-09-10T03:36:32.457 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:32 smithi164 podman[52726]: 2024-09-10 03:36:32.173994649 +0000 UTC m=+1.505026729 container cleanup 3b5c9f00f64dcf30da853c28c3fcfde9568edc12aefa87eae461e6b361a46d4f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, version=8, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , release=754, architecture=x86_64, name=centos-stream, io.buildah.version=1.19.8, ceph=True) 2024-09-10T03:36:32.458 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:32 smithi164 bash[52726]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4 2024-09-10T03:36:32.859 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:32 smithi164 podman[52743]: 2024-09-10 03:36:32.484474737 +0000 UTC m=+0.670960568 container remove 3b5c9f00f64dcf30da853c28c3fcfde9568edc12aefa87eae461e6b361a46d4f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, vcs-type=git, maintainer=Guillaume Abrioux , name=centos-stream, vendor=Red Hat, Inc., distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, release=754, GIT_CLEAN=True, architecture=x86_64, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream) 2024-09-10T03:36:33.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 223739 10.29 MB/sec execute 462 sec latency 1036.698 ms 2024-09-10T03:36:33.139 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:32 smithi164 podman[52828]: 2024-09-10 03:36:32.762113746 +0000 UTC m=+0.017811048 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:36:33.139 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:32 smithi164 podman[52828]: 2024-09-10 03:36:32.885556486 +0000 UTC m=+0.141253787 container create 7b74fe59f28c62a96acbe1a87efce75bda3ae08ec6d76465f2c560d62c4ea63a (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-10T03:36:33.332 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:33 smithi130 ceph-mon[63591]: pgmap v202: 97 pgs: 7 peering, 12 stale+active+clean, 78 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.0 MiB/s wr, 419 op/s 2024-09-10T03:36:33.332 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:33 smithi130 ceph-mon[63591]: osdmap e122: 8 total, 7 up, 8 in 2024-09-10T03:36:33.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:33 smithi164 ceph-mon[49318]: pgmap v202: 97 pgs: 7 peering, 12 stale+active+clean, 78 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.0 MiB/s wr, 419 op/s 2024-09-10T03:36:33.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:33 smithi164 ceph-mon[49318]: osdmap e122: 8 total, 7 up, 8 in 2024-09-10T03:36:33.458 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:33 smithi164 podman[52828]: 2024-09-10 03:36:33.138998262 +0000 UTC m=+0.394695563 container init 7b74fe59f28c62a96acbe1a87efce75bda3ae08ec6d76465f2c560d62c4ea63a (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-10T03:36:33.458 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:33 smithi164 podman[52828]: 2024-09-10 03:36:33.143728105 +0000 UTC m=+0.399425408 container start 7b74fe59f28c62a96acbe1a87efce75bda3ae08ec6d76465f2c560d62c4ea63a (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-deactivate, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-10T03:36:33.458 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:33 smithi164 podman[52828]: 2024-09-10 03:36:33.23922042 +0000 UTC m=+0.494917732 container attach 7b74fe59f28c62a96acbe1a87efce75bda3ae08ec6d76465f2c560d62c4ea63a (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-10T03:36:33.458 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:33 smithi164 podman[52828]: 2024-09-10 03:36:33.362982575 +0000 UTC m=+0.618679884 container died 7b74fe59f28c62a96acbe1a87efce75bda3ae08ec6d76465f2c560d62c4ea63a (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS) 2024-09-10T03:36:34.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 224068 10.29 MB/sec execute 463 sec latency 41.347 ms 2024-09-10T03:36:34.246 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:33 smithi164 podman[52850]: 2024-09-10 03:36:33.99523795 +0000 UTC m=+0.625506310 container remove 7b74fe59f28c62a96acbe1a87efce75bda3ae08ec6d76465f2c560d62c4ea63a (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1) 2024-09-10T03:36:34.246 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:34 smithi164 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-10T03:36:34.246 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:34 smithi164 systemd[1]: Stopped Ceph osd.4 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:36:34.246 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:34 smithi164 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.4.service: Consumed 4min 6.814s CPU time. 2024-09-10T03:36:34.505 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:34 smithi164 systemd[1]: Starting Ceph osd.4 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:36:34.505 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:34 smithi164 podman[52945]: 2024-09-10 03:36:34.38362888 +0000 UTC m=+0.020298659 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:36:34.505 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:34 smithi164 podman[52945]: 2024-09-10 03:36:34.504818429 +0000 UTC m=+0.141488184 container create e9b147ce7db5b9e023f089ea72ee16199744ad5ad915c505245d35ef85411fe2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-activate, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True) 2024-09-10T03:36:34.850 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:34 smithi164 podman[52945]: 2024-09-10 03:36:34.741094981 +0000 UTC m=+0.377764740 container init e9b147ce7db5b9e023f089ea72ee16199744ad5ad915c505245d35ef85411fe2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-activate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1) 2024-09-10T03:36:34.850 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:34 smithi164 podman[52945]: 2024-09-10 03:36:34.745454489 +0000 UTC m=+0.382124248 container start e9b147ce7db5b9e023f089ea72ee16199744ad5ad915c505245d35ef85411fe2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-activate, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1) 2024-09-10T03:36:34.850 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:34 smithi164 podman[52945]: 2024-09-10 03:36:34.849975351 +0000 UTC m=+0.486645121 container attach e9b147ce7db5b9e023f089ea72ee16199744ad5ad915c505245d35ef85411fe2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-10T03:36:35.118 INFO:teuthology.orchestra.run.smithi130.stdout: 5 224307 10.28 MB/sec execute 464 sec latency 69.792 ms 2024-09-10T03:36:35.199 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:34 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-activate[52960]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:36:35.200 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:34 smithi164 bash[52945]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:36:35.200 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:34 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-activate[52960]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:36:35.200 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:34 smithi164 bash[52945]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:36:35.221 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:35 smithi130 ceph-mon[63591]: pgmap v204: 97 pgs: 12 peering, 9 stale+active+clean, 76 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.0 MiB/s wr, 377 op/s 2024-09-10T03:36:35.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:35 smithi164 ceph-mon[49318]: pgmap v204: 97 pgs: 12 peering, 9 stale+active+clean, 76 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.0 MiB/s wr, 377 op/s 2024-09-10T03:36:36.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 224542 10.27 MB/sec execute 465 sec latency 40.931 ms 2024-09-10T03:36:36.497 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:36 smithi130 ceph-mon[63591]: Health check failed: Degraded data redundancy: 202/1668 objects degraded (12.110%), 22 pgs degraded (PG_DEGRADED) 2024-09-10T03:36:36.591 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:36 smithi164 ceph-mon[49318]: Health check failed: Degraded data redundancy: 202/1668 objects degraded (12.110%), 22 pgs degraded (PG_DEGRADED) 2024-09-10T03:36:36.591 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:36 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-activate[52960]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-10T03:36:36.591 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:36 smithi164 bash[52945]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-10T03:36:36.592 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:36 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-activate[52960]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-10T03:36:36.592 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:36 smithi164 bash[52945]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-10T03:36:36.592 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:36 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-activate[52960]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-10T03:36:36.592 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:36 smithi164 bash[52945]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-10T03:36:36.957 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:36 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-activate[52960]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-10T03:36:36.958 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:36 smithi164 bash[52945]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-10T03:36:36.958 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:36 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-activate[52960]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-10T03:36:36.958 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:36 smithi164 bash[52945]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-10T03:36:36.958 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:36 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-activate[52960]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-10T03:36:36.958 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:36 smithi164 bash[52945]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-10T03:36:36.958 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:36 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-activate[52960]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-10T03:36:36.958 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:36 smithi164 bash[52945]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-10T03:36:36.958 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:36 smithi164 podman[52945]: 2024-09-10 03:36:36.619937571 +0000 UTC m=+2.256607337 container died e9b147ce7db5b9e023f089ea72ee16199744ad5ad915c505245d35ef85411fe2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-10T03:36:37.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 224820 10.26 MB/sec execute 466 sec latency 50.334 ms 2024-09-10T03:36:37.268 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:37 smithi130 ceph-mon[63591]: pgmap v205: 97 pgs: 5 active+undersized, 12 peering, 2 stale+active+clean, 22 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 3.3 MiB/s wr, 270 op/s; 202/1668 objects degraded (12.110%) 2024-09-10T03:36:37.586 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:37 smithi164 ceph-mon[49318]: pgmap v205: 97 pgs: 5 active+undersized, 12 peering, 2 stale+active+clean, 22 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 3.3 MiB/s wr, 270 op/s; 202/1668 objects degraded (12.110%) 2024-09-10T03:36:37.586 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:37 smithi164 podman[53104]: 2024-09-10 03:36:37.3808964 +0000 UTC m=+0.754533601 container remove e9b147ce7db5b9e023f089ea72ee16199744ad5ad915c505245d35ef85411fe2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-10T03:36:37.893 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:37 smithi164 podman[53165]: 2024-09-10 03:36:37.488215741 +0000 UTC m=+0.015843139 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:36:37.893 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:37 smithi164 podman[53165]: 2024-09-10 03:36:37.631923041 +0000 UTC m=+0.159550436 container create 5e1e1b5e1266185da08cad1865b597257ead3d1849c141d24ef78718837d5c9f (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-09-10T03:36:38.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 225095 10.26 MB/sec execute 467 sec latency 56.744 ms 2024-09-10T03:36:38.208 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:37 smithi164 podman[53165]: 2024-09-10 03:36:37.89312093 +0000 UTC m=+0.420748319 container init 5e1e1b5e1266185da08cad1865b597257ead3d1849c141d24ef78718837d5c9f (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-10T03:36:38.208 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:37 smithi164 podman[53165]: 2024-09-10 03:36:37.89800917 +0000 UTC m=+0.425636561 container start 5e1e1b5e1266185da08cad1865b597257ead3d1849c141d24ef78718837d5c9f (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1) 2024-09-10T03:36:38.208 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:37 smithi164 ceph-osd[53184]: -- 172.21.15.164:0/3527844186 <== mon.1 v2:172.21.15.164:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56341e8b21e0 con 0x56341da4dc00 2024-09-10T03:36:38.208 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:37 smithi164 bash[53165]: 5e1e1b5e1266185da08cad1865b597257ead3d1849c141d24ef78718837d5c9f 2024-09-10T03:36:38.208 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:37 smithi164 systemd[1]: Started Ceph osd.4 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:36:39.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 225332 10.24 MB/sec execute 468 sec latency 63.680 ms 2024-09-10T03:36:39.287 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:39 smithi130 ceph-mon[63591]: pgmap v206: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.6 MiB/s wr, 438 op/s; 277/1707 objects degraded (16.227%) 2024-09-10T03:36:39.287 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:39 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:39.287 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:39 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:39.287 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:39 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:36:39.429 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:39 smithi164 ceph-mon[49318]: pgmap v206: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.6 MiB/s wr, 438 op/s; 277/1707 objects degraded (16.227%) 2024-09-10T03:36:39.429 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:39 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:39.429 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:39 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:39.429 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:39 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:36:40.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 225744 10.24 MB/sec execute 469 sec latency 42.258 ms 2024-09-10T03:36:40.207 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:39 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[53180]: 2024-09-10T03:36:39.774+0000 7f17798f7640 -1 Falling back to public interface 2024-09-10T03:36:41.068 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:41 smithi130 ceph-mon[63591]: pgmap v207: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.4 KiB/s rd, 2.9 MiB/s wr, 229 op/s; 277/1707 objects degraded (16.227%) 2024-09-10T03:36:41.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 226213 10.24 MB/sec execute 470 sec latency 39.317 ms 2024-09-10T03:36:41.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:41 smithi164 ceph-mon[49318]: pgmap v207: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.4 KiB/s rd, 2.9 MiB/s wr, 229 op/s; 277/1707 objects degraded (16.227%) 2024-09-10T03:36:42.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 226563 10.24 MB/sec execute 471 sec latency 39.628 ms 2024-09-10T03:36:42.132 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:36:42.848 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:42 smithi130 ceph-mon[63591]: pgmap v208: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 437 op/s; 268/1671 objects degraded (16.038%) 2024-09-10T03:36:42.848 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:42 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:42.848 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:42 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:42.848 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:42 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 268/1671 objects degraded (16.038%), 28 pgs degraded (PG_DEGRADED) 2024-09-10T03:36:42.848 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:42 smithi130 ceph-mon[63591]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:36:42.848 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:42 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:42.848 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:42 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:43.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 226912 10.25 MB/sec execute 472 sec latency 42.503 ms 2024-09-10T03:36:43.156 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:36:43.156 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (7m) 39s ago 16m 19.4M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:36:43.156 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (4m) 39s ago 16m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 8c7cfb7e87a1 2024-09-10T03:36:43.156 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (3m) 1s ago 15m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 243c094648b8 2024-09-10T03:36:43.156 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (7m) 39s ago 16m 55.9M - dad864ee21e9 72ba01ac1871 2024-09-10T03:36:43.157 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (10m) 39s ago 10m 37.4M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:36:43.157 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (10m) 1s ago 10m 16.6M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:36:43.157 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (6m) 39s ago 17m 519M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:36:43.157 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283,8765 running (5m) 1s ago 15m 456M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e4034e8e2a6d 2024-09-10T03:36:43.157 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (5m) 39s ago 17m 53.2M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 22e5cf56527a 2024-09-10T03:36:43.157 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (4m) 1s ago 15m 42.0M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 7d80bd0d15b9 2024-09-10T03:36:43.157 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (10m) 39s ago 10m 284M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:36:43.157 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (9m) 1s ago 9m 87.9M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:36:43.157 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (7m) 39s ago 16m 10.7M - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:36:43.157 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (6m) 1s ago 15m 10.6M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:36:43.157 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (3m) 39s ago 14m 353M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b de3cbff0098b 2024-09-10T03:36:43.157 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (2m) 39s ago 14m 273M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f0722a6d1c90 2024-09-10T03:36:43.157 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (97s) 39s ago 13m 209M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b a871d2ccf96e 2024-09-10T03:36:43.158 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (48s) 39s ago 13m 14.2M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f2bc83e91fa2 2024-09-10T03:36:43.158 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (5s) 1s ago 13m 11.8M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 5e1e1b5e1266 2024-09-10T03:36:43.158 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (12m) 1s ago 12m 522M 2704M 17.2.0 e1d6a67b021e f16357ac331e 2024-09-10T03:36:43.158 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (12m) 1s ago 12m 452M 2704M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:36:43.158 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (11m) 1s ago 11m 741M 2704M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:36:43.158 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (5m) 39s ago 15m 54.5M - 2.51.0 1d3b7f56885b f59b02fac5b3 2024-09-10T03:36:43.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:42 smithi164 ceph-mon[49318]: pgmap v208: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 437 op/s; 268/1671 objects degraded (16.038%) 2024-09-10T03:36:43.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:42 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:43.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:42 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:43.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:42 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 268/1671 objects degraded (16.038%), 28 pgs degraded (PG_DEGRADED) 2024-09-10T03:36:43.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:42 smithi164 ceph-mon[49318]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:36:43.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:42 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:43.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:42 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:43.783 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:36:43.784 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:36:43.784 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:36:43.784 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:36:43.784 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:36:43.784 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:36:43.784 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:36:43.784 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:36:43.784 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-10T03:36:43.784 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 4 2024-09-10T03:36:43.784 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:36:43.784 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:36:43.784 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:36:43.785 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:36:43.785 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:36:43.785 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-10T03:36:43.785 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 8 2024-09-10T03:36:43.785 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:36:43.785 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:36:43.887 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:43 smithi130 ceph-mon[63591]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:36:43.887 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:43 smithi130 ceph-mon[63591]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:36:43.887 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:43 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/4258264421' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:36:44.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 227304 10.25 MB/sec execute 473 sec latency 59.548 ms 2024-09-10T03:36:44.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:43 smithi164 ceph-mon[49318]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:36:44.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:43 smithi164 ceph-mon[49318]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:36:44.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:43 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/4258264421' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:36:44.358 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:36:44.359 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:36:44.359 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:36:44.359 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:36:44.359 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-09-10T03:36:44.359 INFO:teuthology.orchestra.run.smithi130.stdout: "mon", 2024-09-10T03:36:44.359 INFO:teuthology.orchestra.run.smithi130.stdout: "crash", 2024-09-10T03:36:44.359 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-09-10T03:36:44.359 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-09-10T03:36:44.359 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "11/23 daemons upgraded", 2024-09-10T03:36:44.359 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading osd daemons", 2024-09-10T03:36:44.359 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:36:44.359 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:36:44.902 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:44 smithi130 ceph-mon[63591]: pgmap v209: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.7 MiB/s wr, 386 op/s; 268/1674 objects degraded (16.010%) 2024-09-10T03:36:45.105 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 268/1674 objects degraded (16.010%), 28 pgs degraded 2024-09-10T03:36:45.105 INFO:teuthology.orchestra.run.smithi130.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-10T03:36:45.105 INFO:teuthology.orchestra.run.smithi130.stdout: osd.4 (root=default,host=smithi164) is down 2024-09-10T03:36:45.105 INFO:teuthology.orchestra.run.smithi130.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 268/1674 objects degraded (16.010%), 28 pgs degraded 2024-09-10T03:36:45.105 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.5 is active+undersized+degraded, acting [7,0] 2024-09-10T03:36:45.105 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-09-10T03:36:45.105 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.b is active+undersized+degraded, acting [7,5] 2024-09-10T03:36:45.106 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.d is active+undersized+degraded, acting [1,3] 2024-09-10T03:36:45.106 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.f is active+undersized+degraded, acting [0,7] 2024-09-10T03:36:45.106 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-09-10T03:36:45.106 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.13 is active+undersized+degraded, acting [0,3] 2024-09-10T03:36:45.106 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-09-10T03:36:45.106 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.1a is active+undersized+degraded, acting [6,7] 2024-09-10T03:36:45.106 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.1f is active+undersized+degraded, acting [0,2] 2024-09-10T03:36:45.106 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.1 is active+undersized+degraded, acting [0,3] 2024-09-10T03:36:45.106 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.3 is active+undersized+degraded, acting [0,6] 2024-09-10T03:36:45.106 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-10T03:36:45.106 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-09-10T03:36:45.106 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.a is active+undersized+degraded, acting [6,1] 2024-09-10T03:36:45.106 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-10T03:36:45.106 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.e is active+undersized+degraded, acting [7,1] 2024-09-10T03:36:45.107 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.f is active+undersized+degraded, acting [7,0] 2024-09-10T03:36:45.107 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.11 is active+undersized+degraded, acting [7,6] 2024-09-10T03:36:45.107 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.13 is active+undersized+degraded, acting [7,2] 2024-09-10T03:36:45.107 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-10T03:36:45.107 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-10T03:36:45.107 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-09-10T03:36:45.107 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-09-10T03:36:45.107 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.1b is active+undersized+degraded, acting [0,7] 2024-09-10T03:36:45.107 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-10T03:36:45.107 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-10T03:36:45.107 INFO:teuthology.orchestra.run.smithi130.stdout: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-09-10T03:36:45.108 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:44 smithi164 ceph-mon[49318]: pgmap v209: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.7 MiB/s wr, 386 op/s; 268/1674 objects degraded (16.010%) 2024-09-10T03:36:45.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 227615 10.25 MB/sec execute 474 sec latency 53.415 ms 2024-09-10T03:36:46.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 227849 10.25 MB/sec execute 475 sec latency 40.562 ms 2024-09-10T03:36:46.170 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:46 smithi130 ceph-mon[63591]: from='client.44291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:36:46.170 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:46 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:46.170 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:46 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:36:46.170 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:46 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/614888682' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:36:46.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:46 smithi164 ceph-mon[49318]: from='client.44291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:36:46.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:46 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:46.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:46 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:36:46.209 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:46 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/614888682' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:36:47.100 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:46 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[53180]: 2024-09-10T03:36:46.912+0000 7f17798f7640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-10T03:36:47.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 228032 10.24 MB/sec execute 476 sec latency 75.680 ms 2024-09-10T03:36:47.209 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:47 smithi130 ceph-mon[63591]: pgmap v210: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 421 op/s; 293/1725 objects degraded (16.986%) 2024-09-10T03:36:47.209 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:47 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:47.209 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:47 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:47.209 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:47 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:36:47.209 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:47 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:36:47.209 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:47 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:47.210 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:47 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:47.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:47 smithi164 ceph-mon[49318]: pgmap v210: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 421 op/s; 293/1725 objects degraded (16.986%) 2024-09-10T03:36:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:47 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:47 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:47 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:36:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:47 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:36:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:47 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:47.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:47 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:36:48.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 228252 10.22 MB/sec execute 477 sec latency 124.679 ms 2024-09-10T03:36:48.254 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:48 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:36:48.254 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:48 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:36:48.254 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:48 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:36:48.254 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:48 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:36:48.254 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:48 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-10T03:36:48.254 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:48 smithi130 ceph-mon[63591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-10T03:36:48.254 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:48 smithi130 ceph-mon[63591]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-10T03:36:48.457 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:48 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[53180]: 2024-09-10T03:36:48.278+0000 7f17798f7640 -1 osd.4 120 log_to_monitors true 2024-09-10T03:36:48.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:48 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:36:48.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:48 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:36:48.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:48 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:36:48.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:48 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:36:48.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:48 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-10T03:36:48.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:48 smithi164 ceph-mon[49318]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-10T03:36:48.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:48 smithi164 ceph-mon[49318]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-10T03:36:49.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 228482 10.21 MB/sec execute 478 sec latency 56.314 ms 2024-09-10T03:36:49.297 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:49 smithi130 ceph-mon[63591]: pgmap v211: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.0 MiB/s wr, 517 op/s; 283/1710 objects degraded (16.550%) 2024-09-10T03:36:49.297 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:49 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 283/1710 objects degraded (16.550%), 28 pgs degraded (PG_DEGRADED) 2024-09-10T03:36:49.297 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:49 smithi130 ceph-mon[63591]: from='osd.4 [v2:172.21.15.164:6800/559100606,v1:172.21.15.164:6801/559100606]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-10T03:36:49.297 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:49 smithi130 ceph-mon[63591]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-10T03:36:49.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:49 smithi164 ceph-mon[49318]: pgmap v211: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.0 MiB/s wr, 517 op/s; 283/1710 objects degraded (16.550%) 2024-09-10T03:36:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:49 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 283/1710 objects degraded (16.550%), 28 pgs degraded (PG_DEGRADED) 2024-09-10T03:36:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:49 smithi164 ceph-mon[49318]: from='osd.4 [v2:172.21.15.164:6800/559100606,v1:172.21.15.164:6801/559100606]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-10T03:36:49.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:49 smithi164 ceph-mon[49318]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-10T03:36:49.957 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:36:49 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[53180]: 2024-09-10T03:36:49.603+0000 7f176d699640 -1 osd.4 120 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-10T03:36:50.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 228639 10.20 MB/sec execute 479 sec latency 144.805 ms 2024-09-10T03:36:50.327 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:50 smithi130 ceph-mon[63591]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-10T03:36:50.327 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:50 smithi130 ceph-mon[63591]: from='osd.4 [v2:172.21.15.164:6800/559100606,v1:172.21.15.164:6801/559100606]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]} : dispatch 2024-09-10T03:36:50.327 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:50 smithi130 ceph-mon[63591]: osdmap e123: 8 total, 7 up, 8 in 2024-09-10T03:36:50.327 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:50 smithi130 ceph-mon[63591]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]} : dispatch 2024-09-10T03:36:50.327 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:50 smithi130 ceph-mon[63591]: from='osd.4 ' entity='osd.4' 2024-09-10T03:36:50.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:50 smithi164 ceph-mon[49318]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-10T03:36:50.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:50 smithi164 ceph-mon[49318]: from='osd.4 [v2:172.21.15.164:6800/559100606,v1:172.21.15.164:6801/559100606]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]} : dispatch 2024-09-10T03:36:50.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:50 smithi164 ceph-mon[49318]: osdmap e123: 8 total, 7 up, 8 in 2024-09-10T03:36:50.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:50 smithi164 ceph-mon[49318]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]} : dispatch 2024-09-10T03:36:50.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:50 smithi164 ceph-mon[49318]: from='osd.4 ' entity='osd.4' 2024-09-10T03:36:51.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 228740 10.18 MB/sec execute 480 sec latency 515.948 ms 2024-09-10T03:36:51.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:51 smithi130 ceph-mon[63591]: pgmap v213: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.9 MiB/s wr, 486 op/s; 283/1710 objects degraded (16.550%) 2024-09-10T03:36:51.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:51 smithi130 ceph-mon[63591]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-10T03:36:51.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:51 smithi130 ceph-mon[63591]: osd.4 [v2:172.21.15.164:6800/559100606,v1:172.21.15.164:6801/559100606] boot 2024-09-10T03:36:51.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:51 smithi130 ceph-mon[63591]: osdmap e124: 8 total, 8 up, 8 in 2024-09-10T03:36:51.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:51 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-10T03:36:51.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:51 smithi164 ceph-mon[49318]: pgmap v213: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.9 MiB/s wr, 486 op/s; 283/1710 objects degraded (16.550%) 2024-09-10T03:36:51.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:51 smithi164 ceph-mon[49318]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-10T03:36:51.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:51 smithi164 ceph-mon[49318]: osd.4 [v2:172.21.15.164:6800/559100606,v1:172.21.15.164:6801/559100606] boot 2024-09-10T03:36:51.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:51 smithi164 ceph-mon[49318]: osdmap e124: 8 total, 8 up, 8 in 2024-09-10T03:36:51.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:51 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-10T03:36:52.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 228847 10.16 MB/sec execute 481 sec latency 1103.273 ms 2024-09-10T03:36:52.868 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:52 smithi130 ceph-mon[63591]: osdmap e125: 8 total, 8 up, 8 in 2024-09-10T03:36:52.868 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:52 smithi130 ceph-mon[63591]: pgmap v216: 97 pgs: 2 activating+degraded, 2 remapped+peering, 14 peering, 11 active+undersized, 10 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 4.2 MiB/s wr, 500 op/s; 126/1620 objects degraded (7.778%) 2024-09-10T03:36:52.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:52 smithi164 ceph-mon[49318]: osdmap e125: 8 total, 8 up, 8 in 2024-09-10T03:36:52.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:52 smithi164 ceph-mon[49318]: pgmap v216: 97 pgs: 2 activating+degraded, 2 remapped+peering, 14 peering, 11 active+undersized, 10 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 4.2 MiB/s wr, 500 op/s; 126/1620 objects degraded (7.778%) 2024-09-10T03:36:53.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 228936 10.15 MB/sec execute 482 sec latency 1080.094 ms 2024-09-10T03:36:53.655 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:53 smithi130 ceph-mon[63591]: osdmap e126: 8 total, 8 up, 8 in 2024-09-10T03:36:53.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:53 smithi164 ceph-mon[49318]: osdmap e126: 8 total, 8 up, 8 in 2024-09-10T03:36:54.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 229168 10.14 MB/sec execute 483 sec latency 468.207 ms 2024-09-10T03:36:54.709 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:54 smithi130 ceph-mon[63591]: osdmap e127: 8 total, 8 up, 8 in 2024-09-10T03:36:54.709 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:54 smithi130 ceph-mon[63591]: pgmap v219: 97 pgs: 2 activating+degraded, 3 remapped+peering, 14 peering, 10 active+undersized, 9 active+undersized+degraded, 59 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.3 MiB/s wr, 375 op/s; 109/1557 objects degraded (7.001%) 2024-09-10T03:36:54.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:54 smithi164 ceph-mon[49318]: osdmap e127: 8 total, 8 up, 8 in 2024-09-10T03:36:54.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:54 smithi164 ceph-mon[49318]: pgmap v219: 97 pgs: 2 activating+degraded, 3 remapped+peering, 14 peering, 10 active+undersized, 9 active+undersized+degraded, 59 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.3 MiB/s wr, 375 op/s; 109/1557 objects degraded (7.001%) 2024-09-10T03:36:55.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 229397 10.13 MB/sec execute 484 sec latency 1008.962 ms 2024-09-10T03:36:55.813 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:55 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 109/1557 objects degraded (7.001%), 11 pgs degraded (PG_DEGRADED) 2024-09-10T03:36:55.813 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:55 smithi130 ceph-mon[63591]: osdmap e128: 8 total, 8 up, 8 in 2024-09-10T03:36:55.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:55 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 109/1557 objects degraded (7.001%), 11 pgs degraded (PG_DEGRADED) 2024-09-10T03:36:55.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:55 smithi164 ceph-mon[49318]: osdmap e128: 8 total, 8 up, 8 in 2024-09-10T03:36:56.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 229704 10.12 MB/sec execute 485 sec latency 404.027 ms 2024-09-10T03:36:56.860 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:56 smithi130 ceph-mon[63591]: osdmap e129: 8 total, 8 up, 8 in 2024-09-10T03:36:56.860 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:56 smithi130 ceph-mon[63591]: pgmap v222: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 activating+degraded, 2 remapped+peering, 3 peering, 89 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.4 MiB/s wr, 300 op/s; 28/1545 objects degraded (1.812%); 432 KiB/s, 66 objects/s recovering 2024-09-10T03:36:56.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:56 smithi164 ceph-mon[49318]: osdmap e129: 8 total, 8 up, 8 in 2024-09-10T03:36:56.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:56 smithi164 ceph-mon[49318]: pgmap v222: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 activating+degraded, 2 remapped+peering, 3 peering, 89 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.4 MiB/s wr, 300 op/s; 28/1545 objects degraded (1.812%); 432 KiB/s, 66 objects/s recovering 2024-09-10T03:36:57.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 229993 10.11 MB/sec execute 486 sec latency 952.674 ms 2024-09-10T03:36:57.871 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:57 smithi130 ceph-mon[63591]: osdmap e130: 8 total, 8 up, 8 in 2024-09-10T03:36:57.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:57 smithi164 ceph-mon[49318]: osdmap e130: 8 total, 8 up, 8 in 2024-09-10T03:36:58.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 230287 10.11 MB/sec execute 487 sec latency 934.799 ms 2024-09-10T03:36:58.887 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:58 smithi130 ceph-mon[63591]: osdmap e131: 8 total, 8 up, 8 in 2024-09-10T03:36:58.887 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:58 smithi130 ceph-mon[63591]: pgmap v225: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 3 peering, 93 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.3 MiB/s wr, 316 op/s; 6/1584 objects degraded (0.379%); 1.7 MiB/s, 37 keys/s, 77 objects/s recovering 2024-09-10T03:36:58.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:58 smithi164 ceph-mon[49318]: osdmap e131: 8 total, 8 up, 8 in 2024-09-10T03:36:58.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:58 smithi164 ceph-mon[49318]: pgmap v225: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 3 peering, 93 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.3 MiB/s wr, 316 op/s; 6/1584 objects degraded (0.379%); 1.7 MiB/s, 37 keys/s, 77 objects/s recovering 2024-09-10T03:36:59.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 230625 10.11 MB/sec execute 488 sec latency 80.241 ms 2024-09-10T03:36:59.915 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:36:59 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 6/1590 objects degraded (0.377%), 1 pg degraded (PG_DEGRADED) 2024-09-10T03:37:00.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 230972 10.12 MB/sec execute 489 sec latency 76.168 ms 2024-09-10T03:37:00.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:36:59 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 6/1590 objects degraded (0.377%), 1 pg degraded (PG_DEGRADED) 2024-09-10T03:37:01.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 231248 10.12 MB/sec execute 490 sec latency 64.090 ms 2024-09-10T03:37:01.260 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:01 smithi130 ceph-mon[63591]: pgmap v226: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 95 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.0 MiB/s wr, 251 op/s; 6/1590 objects degraded (0.377%); 1.1 MiB/s, 29 keys/s, 45 objects/s recovering 2024-09-10T03:37:01.260 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:01.260 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:37:01.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:01 smithi164 ceph-mon[49318]: pgmap v226: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 95 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.0 MiB/s wr, 251 op/s; 6/1590 objects degraded (0.377%); 1.1 MiB/s, 29 keys/s, 45 objects/s recovering 2024-09-10T03:37:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:37:02.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 231535 10.12 MB/sec execute 491 sec latency 47.560 ms 2024-09-10T03:37:02.282 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:02 smithi130 ceph-mon[63591]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/1590 objects degraded (0.377%), 1 pg degraded) 2024-09-10T03:37:02.282 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:02 smithi130 ceph-mon[63591]: Cluster is now healthy 2024-09-10T03:37:02.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:02 smithi164 ceph-mon[49318]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/1590 objects degraded (0.377%), 1 pg degraded) 2024-09-10T03:37:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:02 smithi164 ceph-mon[49318]: Cluster is now healthy 2024-09-10T03:37:03.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 231850 10.11 MB/sec execute 492 sec latency 39.594 ms 2024-09-10T03:37:03.293 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:03 smithi130 ceph-mon[63591]: pgmap v227: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.5 MiB/s wr, 509 op/s; 833 KiB/s, 24 keys/s, 31 objects/s recovering 2024-09-10T03:37:03.293 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:03 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-10T03:37:03.293 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:03 smithi130 ceph-mon[63591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-10T03:37:03.293 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:03 smithi130 ceph-mon[63591]: Upgrade: osd.5 is safe to restart 2024-09-10T03:37:03.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:03 smithi164 ceph-mon[49318]: pgmap v227: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.5 MiB/s wr, 509 op/s; 833 KiB/s, 24 keys/s, 31 objects/s recovering 2024-09-10T03:37:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:03 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-10T03:37:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:03 smithi164 ceph-mon[49318]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-10T03:37:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:03 smithi164 ceph-mon[49318]: Upgrade: osd.5 is safe to restart 2024-09-10T03:37:04.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 232065 10.10 MB/sec execute 493 sec latency 62.878 ms 2024-09-10T03:37:04.836 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:04 smithi130 ceph-mon[63591]: Upgrade: Updating osd.5 2024-09-10T03:37:04.836 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:04 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:04.836 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:04 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-10T03:37:04.836 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:04 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:37:04.836 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:04 smithi130 ceph-mon[63591]: Deploying daemon osd.5 on smithi164 2024-09-10T03:37:04.836 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:04 smithi130 ceph-mon[63591]: pgmap v228: 97 pgs: 97 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.4 MiB/s wr, 291 op/s; 633 KiB/s, 18 keys/s, 6 objects/s recovering 2024-09-10T03:37:05.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 232385 10.09 MB/sec execute 494 sec latency 44.314 ms 2024-09-10T03:37:05.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:04 smithi164 ceph-mon[49318]: Upgrade: Updating osd.5 2024-09-10T03:37:05.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:04 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:05.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:04 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-10T03:37:05.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:04 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:37:05.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:04 smithi164 ceph-mon[49318]: Deploying daemon osd.5 on smithi164 2024-09-10T03:37:05.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:04 smithi164 ceph-mon[49318]: pgmap v228: 97 pgs: 97 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.4 MiB/s wr, 291 op/s; 633 KiB/s, 18 keys/s, 6 objects/s recovering 2024-09-10T03:37:06.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 232719 10.09 MB/sec execute 495 sec latency 63.363 ms 2024-09-10T03:37:06.861 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:06 smithi130 ceph-mon[63591]: pgmap v229: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.1 MiB/s wr, 320 op/s; 99 KiB/s, 3 objects/s recovering 2024-09-10T03:37:06.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:06 smithi164 ceph-mon[49318]: pgmap v229: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.1 MiB/s wr, 320 op/s; 99 KiB/s, 3 objects/s recovering 2024-09-10T03:37:07.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 233067 10.07 MB/sec execute 496 sec latency 54.237 ms 2024-09-10T03:37:07.452 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:07 smithi164 systemd[1]: Stopping Ceph osd.5 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:37:07.707 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:07 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[32876]: 2024-09-10T03:37:07.450+0000 7fa748647700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:37:07.708 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:07 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[32876]: 2024-09-10T03:37:07.450+0000 7fa748647700 -1 osd.5 131 *** Got signal Terminated *** 2024-09-10T03:37:07.708 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:07 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[32876]: 2024-09-10T03:37:07.450+0000 7fa748647700 -1 osd.5 131 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-10T03:37:08.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:07 smithi130 ceph-mon[63591]: osd.5 marked itself down and dead 2024-09-10T03:37:08.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 233309 10.07 MB/sec execute 497 sec latency 256.010 ms 2024-09-10T03:37:08.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:07 smithi164 ceph-mon[49318]: osd.5 marked itself down and dead 2024-09-10T03:37:08.810 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:08 smithi164 podman[54754]: 2024-09-10 03:37:08.518978453 +0000 UTC m=+1.188381583 container died f16357ac331ee7293ab2b2093081ebcf1df8aeb77ec03c4054740982cb3addc6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, name=centos-stream, io.openshift.expose-services=) 2024-09-10T03:37:09.111 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:08 smithi130 ceph-mon[63591]: pgmap v230: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 470 op/s; 0 B/s, 1 objects/s recovering 2024-09-10T03:37:09.111 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:08 smithi130 ceph-mon[63591]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-10T03:37:09.111 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:08 smithi130 ceph-mon[63591]: osdmap e132: 8 total, 7 up, 8 in 2024-09-10T03:37:09.119 INFO:teuthology.orchestra.run.smithi130.stdout: 5 233358 10.05 MB/sec execute 498 sec latency 1034.389 ms 2024-09-10T03:37:09.203 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:08 smithi164 ceph-mon[49318]: pgmap v230: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 470 op/s; 0 B/s, 1 objects/s recovering 2024-09-10T03:37:09.203 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:08 smithi164 ceph-mon[49318]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-10T03:37:09.203 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:08 smithi164 ceph-mon[49318]: osdmap e132: 8 total, 7 up, 8 in 2024-09-10T03:37:09.203 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:08 smithi164 podman[54754]: 2024-09-10 03:37:08.908862863 +0000 UTC m=+1.578265991 container cleanup f16357ac331ee7293ab2b2093081ebcf1df8aeb77ec03c4054740982cb3addc6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.openshift.tags=base centos centos-stream, architecture=x86_64, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, release=754, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, distribution-scope=public, io.openshift.expose-services=, com.redhat.component=centos-stream-container) 2024-09-10T03:37:09.203 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:08 smithi164 bash[54754]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5 2024-09-10T03:37:09.457 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:09 smithi164 podman[54770]: 2024-09-10 03:37:09.202150079 +0000 UTC m=+0.680603312 container remove f16357ac331ee7293ab2b2093081ebcf1df8aeb77ec03c4054740982cb3addc6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, RELEASE=HEAD, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, ceph=True, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-10T03:37:09.839 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:09 smithi164 podman[54856]: 2024-09-10 03:37:09.456363286 +0000 UTC m=+0.018509073 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:37:09.840 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:09 smithi164 podman[54856]: 2024-09-10 03:37:09.58666328 +0000 UTC m=+0.148809069 container create 237a45f07aca72a0f833dce18d8182b406d25d1474c49064e7f0f6acc2ba4466 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-deactivate, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True) 2024-09-10T03:37:09.840 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:09 smithi164 podman[54856]: 2024-09-10 03:37:09.839482123 +0000 UTC m=+0.401627912 container init 237a45f07aca72a0f833dce18d8182b406d25d1474c49064e7f0f6acc2ba4466 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-10T03:37:10.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 233660 10.05 MB/sec execute 499 sec latency 47.853 ms 2024-09-10T03:37:10.133 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:09 smithi130 ceph-mon[63591]: osdmap e133: 8 total, 7 up, 8 in 2024-09-10T03:37:10.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:09 smithi164 ceph-mon[49318]: osdmap e133: 8 total, 7 up, 8 in 2024-09-10T03:37:10.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:09 smithi164 podman[54856]: 2024-09-10 03:37:09.843188947 +0000 UTC m=+0.405334736 container start 237a45f07aca72a0f833dce18d8182b406d25d1474c49064e7f0f6acc2ba4466 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-10T03:37:10.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:09 smithi164 podman[54856]: 2024-09-10 03:37:09.931417666 +0000 UTC m=+0.493563465 container attach 237a45f07aca72a0f833dce18d8182b406d25d1474c49064e7f0f6acc2ba4466 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-10T03:37:10.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:10 smithi164 podman[54856]: 2024-09-10 03:37:10.076938502 +0000 UTC m=+0.639084306 container died 237a45f07aca72a0f833dce18d8182b406d25d1474c49064e7f0f6acc2ba4466 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-10T03:37:10.896 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:10 smithi130 ceph-mon[63591]: pgmap v233: 97 pgs: 2 active+undersized, 9 stale+active+clean, 3 active+undersized+degraded, 83 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.7 MiB/s wr, 362 op/s; 22/1659 objects degraded (1.326%) 2024-09-10T03:37:10.896 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:10 smithi130 ceph-mon[63591]: Health check failed: Degraded data redundancy: 22/1659 objects degraded (1.326%), 3 pgs degraded (PG_DEGRADED) 2024-09-10T03:37:10.896 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:11.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 234094 10.05 MB/sec execute 500 sec latency 41.841 ms 2024-09-10T03:37:11.180 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:10 smithi164 ceph-mon[49318]: pgmap v233: 97 pgs: 2 active+undersized, 9 stale+active+clean, 3 active+undersized+degraded, 83 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.7 MiB/s wr, 362 op/s; 22/1659 objects degraded (1.326%) 2024-09-10T03:37:11.180 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:10 smithi164 ceph-mon[49318]: Health check failed: Degraded data redundancy: 22/1659 objects degraded (1.326%), 3 pgs degraded (PG_DEGRADED) 2024-09-10T03:37:11.180 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:11.180 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:10 smithi164 podman[54880]: 2024-09-10 03:37:10.886425432 +0000 UTC m=+0.802629620 container remove 237a45f07aca72a0f833dce18d8182b406d25d1474c49064e7f0f6acc2ba4466 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_CLEAN=True) 2024-09-10T03:37:11.181 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:10 smithi164 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-10T03:37:11.181 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:10 smithi164 systemd[1]: Stopped Ceph osd.5 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:37:11.181 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:10 smithi164 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.5.service: Consumed 4min 37.225s CPU time. 2024-09-10T03:37:11.454 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:11 smithi164 systemd[1]: Starting Ceph osd.5 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:37:11.455 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:11 smithi164 podman[54977]: 2024-09-10 03:37:11.325105072 +0000 UTC m=+0.021027278 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:37:11.455 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:11 smithi164 podman[54977]: 2024-09-10 03:37:11.454344654 +0000 UTC m=+0.150266833 container create 342f65ec4cc051b28957158c07d46b6456e65e523be7222f89da4bd3ddeeac8d (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1) 2024-09-10T03:37:12.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 234367 10.05 MB/sec execute 501 sec latency 53.528 ms 2024-09-10T03:37:12.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:11 smithi164 podman[54977]: 2024-09-10 03:37:11.707662377 +0000 UTC m=+0.403584573 container init 342f65ec4cc051b28957158c07d46b6456e65e523be7222f89da4bd3ddeeac8d (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-activate, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-09-10T03:37:12.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:11 smithi164 podman[54977]: 2024-09-10 03:37:11.712391429 +0000 UTC m=+0.408313608 container start 342f65ec4cc051b28957158c07d46b6456e65e523be7222f89da4bd3ddeeac8d (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-activate, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-10T03:37:12.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:11 smithi164 podman[54977]: 2024-09-10 03:37:11.807715466 +0000 UTC m=+0.503637659 container attach 342f65ec4cc051b28957158c07d46b6456e65e523be7222f89da4bd3ddeeac8d (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-10T03:37:12.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:11 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-activate[54992]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:37:12.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:11 smithi164 bash[54977]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:37:12.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:11 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-activate[54992]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:37:12.209 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:11 smithi164 bash[54977]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:37:12.974 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:12 smithi130 ceph-mon[63591]: pgmap v234: 97 pgs: 8 active+undersized, 3 stale+active+clean, 18 active+undersized+degraded, 68 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.0 MiB/s wr, 441 op/s; 124/1614 objects degraded (7.683%) 2024-09-10T03:37:13.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 234697 10.07 MB/sec execute 502 sec latency 46.497 ms 2024-09-10T03:37:13.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:12 smithi164 ceph-mon[49318]: pgmap v234: 97 pgs: 8 active+undersized, 3 stale+active+clean, 18 active+undersized+degraded, 68 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.0 MiB/s wr, 441 op/s; 124/1614 objects degraded (7.683%) 2024-09-10T03:37:13.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:12 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-activate[54992]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-10T03:37:13.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:12 smithi164 bash[54977]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-10T03:37:13.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:12 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-activate[54992]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-10T03:37:13.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:12 smithi164 bash[54977]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-10T03:37:13.612 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:13 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-activate[54992]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-10T03:37:13.613 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:13 smithi164 bash[54977]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-10T03:37:13.613 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:13 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-activate[54992]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-10T03:37:13.613 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:13 smithi164 bash[54977]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-10T03:37:13.613 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:13 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-activate[54992]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-10T03:37:13.613 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:13 smithi164 bash[54977]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-10T03:37:13.613 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:13 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-activate[54992]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-10T03:37:13.613 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:13 smithi164 bash[54977]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-10T03:37:13.613 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:13 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-activate[54992]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-10T03:37:13.613 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:13 smithi164 bash[54977]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-10T03:37:13.613 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:13 smithi164 podman[54977]: 2024-09-10 03:37:13.282706051 +0000 UTC m=+1.978628234 container died 342f65ec4cc051b28957158c07d46b6456e65e523be7222f89da4bd3ddeeac8d (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-activate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-10T03:37:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 234976 10.06 MB/sec execute 503 sec latency 37.587 ms 2024-09-10T03:37:14.189 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:13 smithi164 podman[55124]: 2024-09-10 03:37:13.937826357 +0000 UTC m=+0.645412358 container remove 342f65ec4cc051b28957158c07d46b6456e65e523be7222f89da4bd3ddeeac8d (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-activate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-10T03:37:14.189 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:14 smithi164 podman[55186]: 2024-09-10 03:37:14.06177739 +0000 UTC m=+0.021713652 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:37:14.189 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:14 smithi164 podman[55186]: 2024-09-10 03:37:14.188795103 +0000 UTC m=+0.148731407 container create 76ab1aa88b44a55155f9e824a08b98ce81e765f09872ea21b4ffdd8ea0e7e0b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True) 2024-09-10T03:37:14.443 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:14 smithi164 podman[55186]: 2024-09-10 03:37:14.416199573 +0000 UTC m=+0.376135834 container init 76ab1aa88b44a55155f9e824a08b98ce81e765f09872ea21b4ffdd8ea0e7e0b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1) 2024-09-10T03:37:14.443 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:14 smithi164 podman[55186]: 2024-09-10 03:37:14.419550167 +0000 UTC m=+0.379486428 container start 76ab1aa88b44a55155f9e824a08b98ce81e765f09872ea21b4ffdd8ea0e7e0b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-10T03:37:14.708 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:14 smithi164 ceph-osd[55205]: -- 172.21.15.164:0/609857218 <== mon.1 v2:172.21.15.164:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563ce31301e0 con 0x563ce30c0000 2024-09-10T03:37:14.708 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:14 smithi164 bash[55186]: 76ab1aa88b44a55155f9e824a08b98ce81e765f09872ea21b4ffdd8ea0e7e0b8 2024-09-10T03:37:14.708 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:14 smithi164 systemd[1]: Started Ceph osd.5 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:37:15.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:14 smithi130 ceph-mon[63591]: pgmap v235: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 3.6 MiB/s wr, 491 op/s; 172/1590 objects degraded (10.818%) 2024-09-10T03:37:15.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:14 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:15.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:14 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:15.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:14 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:37:15.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 235232 10.06 MB/sec execute 504 sec latency 52.986 ms 2024-09-10T03:37:15.199 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:14 smithi164 ceph-mon[49318]: pgmap v235: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 3.6 MiB/s wr, 491 op/s; 172/1590 objects degraded (10.818%) 2024-09-10T03:37:15.199 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:14 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:15.199 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:14 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:15.199 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:14 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:37:15.662 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:37:16.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 235440 10.04 MB/sec execute 505 sec latency 56.243 ms 2024-09-10T03:37:16.215 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:16 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:16.215 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:16 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:37:16.215 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:16 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 172/1590 objects degraded (10.818%), 23 pgs degraded (PG_DEGRADED) 2024-09-10T03:37:16.350 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:16 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:37:16.317+0000 7fc3a8af9640 -1 Falling back to public interface 2024-09-10T03:37:16.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:16 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:16.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:16 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:37:16.350 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:16 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 172/1590 objects degraded (10.818%), 23 pgs degraded (PG_DEGRADED) 2024-09-10T03:37:16.635 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:37:16.635 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (8m) 72s ago 17m 19.4M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:37:16.635 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (4m) 72s ago 16m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 8c7cfb7e87a1 2024-09-10T03:37:16.635 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (4m) 34s ago 16m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 243c094648b8 2024-09-10T03:37:16.635 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (7m) 72s ago 16m 55.9M - dad864ee21e9 72ba01ac1871 2024-09-10T03:37:16.635 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (10m) 72s ago 10m 37.4M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:37:16.635 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (10m) 34s ago 10m 16.6M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:37:16.635 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (7m) 72s ago 18m 519M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:37:16.636 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283,8765 running (6m) 34s ago 16m 456M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e4034e8e2a6d 2024-09-10T03:37:16.636 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (5m) 72s ago 18m 53.2M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 22e5cf56527a 2024-09-10T03:37:16.636 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (5m) 34s ago 16m 42.0M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 7d80bd0d15b9 2024-09-10T03:37:16.636 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (10m) 72s ago 10m 284M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:37:16.636 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (10m) 34s ago 10m 87.9M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:37:16.636 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (7m) 72s ago 16m 10.7M - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:37:16.636 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (7m) 34s ago 16m 10.6M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:37:16.636 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (3m) 72s ago 15m 353M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b de3cbff0098b 2024-09-10T03:37:16.636 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (3m) 72s ago 14m 273M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f0722a6d1c90 2024-09-10T03:37:16.636 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (2m) 72s ago 14m 209M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b a871d2ccf96e 2024-09-10T03:37:16.636 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (81s) 72s ago 14m 14.2M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f2bc83e91fa2 2024-09-10T03:37:16.636 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (39s) 34s ago 13m 11.8M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 5e1e1b5e1266 2024-09-10T03:37:16.637 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 starting - - - 2704M 2024-09-10T03:37:16.637 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (12m) 34s ago 12m 452M 2704M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:37:16.637 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (12m) 34s ago 12m 741M 2704M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:37:16.637 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (6m) 72s ago 15m 54.5M - 2.51.0 1d3b7f56885b f59b02fac5b3 2024-09-10T03:37:17.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 235660 10.03 MB/sec execute 506 sec latency 47.773 ms 2024-09-10T03:37:17.163 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:37:17.164 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:37:17.164 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:37:17.164 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:37:17.164 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:37:17.164 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:37:17.164 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:37:17.164 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:37:17.164 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-10T03:37:17.164 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 5 2024-09-10T03:37:17.164 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:37:17.164 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:37:17.165 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:37:17.165 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:37:17.165 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:37:17.165 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-10T03:37:17.165 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 9 2024-09-10T03:37:17.165 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:37:17.165 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:37:17.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:17 smithi164 ceph-mon[49318]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:37:17.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:17 smithi164 ceph-mon[49318]: pgmap v236: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.8 MiB/s wr, 300 op/s; 188/1665 objects degraded (11.291%) 2024-09-10T03:37:17.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:17 smithi164 ceph-mon[49318]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:37:17.232 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:17 smithi130 ceph-mon[63591]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:37:17.232 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:17 smithi130 ceph-mon[63591]: pgmap v236: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.8 MiB/s wr, 300 op/s; 188/1665 objects degraded (11.291%) 2024-09-10T03:37:17.232 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:17 smithi130 ceph-mon[63591]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:37:17.574 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:37:17.574 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:37:17.574 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:37:17.574 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:37:17.575 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-09-10T03:37:17.575 INFO:teuthology.orchestra.run.smithi130.stdout: "mon", 2024-09-10T03:37:17.575 INFO:teuthology.orchestra.run.smithi130.stdout: "crash", 2024-09-10T03:37:17.575 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-09-10T03:37:17.575 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-09-10T03:37:17.575 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "11/23 daemons upgraded", 2024-09-10T03:37:17.575 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading osd daemons", 2024-09-10T03:37:17.575 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:37:17.575 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:37:18.117 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 195/1713 objects degraded (11.384%), 23 pgs degraded 2024-09-10T03:37:18.117 INFO:teuthology.orchestra.run.smithi130.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-10T03:37:18.117 INFO:teuthology.orchestra.run.smithi130.stdout: osd.5 (root=default,host=smithi164) is down 2024-09-10T03:37:18.117 INFO:teuthology.orchestra.run.smithi130.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 195/1713 objects degraded (11.384%), 23 pgs degraded 2024-09-10T03:37:18.117 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-10T03:37:18.118 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-09-10T03:37:18.118 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.b is active+undersized+degraded, acting [7,4] 2024-09-10T03:37:18.118 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.c is active+undersized+degraded, acting [2,0] 2024-09-10T03:37:18.118 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-09-10T03:37:18.118 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-09-10T03:37:18.118 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-09-10T03:37:18.118 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-09-10T03:37:18.118 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-09-10T03:37:18.118 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-09-10T03:37:18.118 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-09-10T03:37:18.118 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-09-10T03:37:18.118 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-09-10T03:37:18.118 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-09-10T03:37:18.119 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-09-10T03:37:18.119 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-09-10T03:37:18.119 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-10T03:37:18.119 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-10T03:37:18.119 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-09-10T03:37:18.119 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-09-10T03:37:18.119 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-09-10T03:37:18.119 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-09-10T03:37:18.119 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-10T03:37:18.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 235940 10.02 MB/sec execute 507 sec latency 75.888 ms 2024-09-10T03:37:18.245 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:18 smithi130 ceph-mon[63591]: from='client.44307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:37:18.245 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:18 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/1459516072' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:37:18.310 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:18 smithi164 ceph-mon[49318]: from='client.44307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:37:18.310 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:18 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/1459516072' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:37:19.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 236263 10.02 MB/sec execute 508 sec latency 36.856 ms 2024-09-10T03:37:19.266 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:19 smithi130 ceph-mon[63591]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:37:19.266 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:19 smithi130 ceph-mon[63591]: pgmap v237: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.2 MiB/s wr, 386 op/s; 195/1713 objects degraded (11.384%) 2024-09-10T03:37:19.266 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:19 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/150168792' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:37:19.266 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:19 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:19.266 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:19 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:19.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:19 smithi164 ceph-mon[49318]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:37:19.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:19 smithi164 ceph-mon[49318]: pgmap v237: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.2 MiB/s wr, 386 op/s; 195/1713 objects degraded (11.384%) 2024-09-10T03:37:19.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:19 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/150168792' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:37:19.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:19 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:19.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:19 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:20.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 236598 10.01 MB/sec execute 509 sec latency 59.481 ms 2024-09-10T03:37:20.534 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:20 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:20.534 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:20 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:20.534 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:20 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 195/1713 objects degraded (11.384%), 23 pgs degraded (PG_DEGRADED) 2024-09-10T03:37:20.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:20 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:20.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:20 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:20.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:20 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 195/1713 objects degraded (11.384%), 23 pgs degraded (PG_DEGRADED) 2024-09-10T03:37:21.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 236992 10.01 MB/sec execute 510 sec latency 50.380 ms 2024-09-10T03:37:21.366 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:21 smithi164 ceph-mon[49318]: pgmap v238: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.4 MiB/s wr, 438 op/s; 199/1710 objects degraded (11.637%) 2024-09-10T03:37:21.581 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:21 smithi130 ceph-mon[63591]: pgmap v238: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.4 MiB/s wr, 438 op/s; 199/1710 objects degraded (11.637%) 2024-09-10T03:37:22.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 237355 10.00 MB/sec execute 511 sec latency 60.140 ms 2024-09-10T03:37:23.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 237716 10.01 MB/sec execute 512 sec latency 58.053 ms 2024-09-10T03:37:23.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:23 smithi130 ceph-mon[63591]: pgmap v239: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.6 MiB/s wr, 473 op/s; 188/1653 objects degraded (11.373%) 2024-09-10T03:37:23.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:23 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:23.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:23 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:23.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:23 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:37:23.363 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:23 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:37:23.364 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:23 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:23.364 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:23 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:23.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:23 smithi164 ceph-mon[49318]: pgmap v239: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.6 MiB/s wr, 473 op/s; 188/1653 objects degraded (11.373%) 2024-09-10T03:37:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:23 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:23 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:23 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:37:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:23 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:37:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:23 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:23.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:23 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:24.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 238040 10.01 MB/sec execute 513 sec latency 48.497 ms 2024-09-10T03:37:24.378 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:24 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:37:24.378 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:24 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:37:24.378 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:24 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:37:24.378 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:24 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:37:24.378 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:24 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-10T03:37:24.707 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:24 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:37:24.482+0000 7fc3a8af9640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-10T03:37:24.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:24 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:37:24.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:24 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:37:24.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:24 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:37:24.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:24 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:37:24.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:24 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-10T03:37:25.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 238363 10.02 MB/sec execute 514 sec latency 50.161 ms 2024-09-10T03:37:25.409 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:25 smithi130 ceph-mon[63591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-10T03:37:25.409 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:25 smithi130 ceph-mon[63591]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-10T03:37:25.409 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:25 smithi130 ceph-mon[63591]: pgmap v240: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 427 op/s; 184/1641 objects degraded (11.213%) 2024-09-10T03:37:25.409 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:25 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 184/1641 objects degraded (11.213%), 23 pgs degraded (PG_DEGRADED) 2024-09-10T03:37:25.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:25 smithi164 ceph-mon[49318]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-10T03:37:25.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:25 smithi164 ceph-mon[49318]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-10T03:37:25.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:25 smithi164 ceph-mon[49318]: pgmap v240: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 427 op/s; 184/1641 objects degraded (11.213%) 2024-09-10T03:37:25.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:25 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 184/1641 objects degraded (11.213%), 23 pgs degraded (PG_DEGRADED) 2024-09-10T03:37:26.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 238651 10.02 MB/sec execute 515 sec latency 78.008 ms 2024-09-10T03:37:26.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:25 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:37:25.714+0000 7fc3a8af9640 -1 osd.5 131 log_to_monitors true 2024-09-10T03:37:26.439 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:26 smithi130 ceph-mon[63591]: from='osd.5 [v2:172.21.15.164:6808/395559117,v1:172.21.15.164:6809/395559117]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-10T03:37:26.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:26 smithi164 ceph-mon[49318]: from='osd.5 [v2:172.21.15.164:6808/395559117,v1:172.21.15.164:6809/395559117]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-10T03:37:26.708 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:37:26 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:37:26.365+0000 7fc3a08a3640 -1 osd.5 131 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-10T03:37:27.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 238876 10.01 MB/sec execute 516 sec latency 79.281 ms 2024-09-10T03:37:27.480 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:27 smithi130 ceph-mon[63591]: pgmap v241: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.0 MiB/s wr, 446 op/s; 181/1677 objects degraded (10.793%) 2024-09-10T03:37:27.480 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:27 smithi130 ceph-mon[63591]: from='osd.5 [v2:172.21.15.164:6808/395559117,v1:172.21.15.164:6809/395559117]' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-10T03:37:27.480 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:27 smithi130 ceph-mon[63591]: osdmap e134: 8 total, 7 up, 8 in 2024-09-10T03:37:27.480 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:27 smithi130 ceph-mon[63591]: from='osd.5 [v2:172.21.15.164:6808/395559117,v1:172.21.15.164:6809/395559117]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]} : dispatch 2024-09-10T03:37:27.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:27 smithi164 ceph-mon[49318]: pgmap v241: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.0 MiB/s wr, 446 op/s; 181/1677 objects degraded (10.793%) 2024-09-10T03:37:27.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:27 smithi164 ceph-mon[49318]: from='osd.5 [v2:172.21.15.164:6808/395559117,v1:172.21.15.164:6809/395559117]' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-10T03:37:27.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:27 smithi164 ceph-mon[49318]: osdmap e134: 8 total, 7 up, 8 in 2024-09-10T03:37:27.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:27 smithi164 ceph-mon[49318]: from='osd.5 [v2:172.21.15.164:6808/395559117,v1:172.21.15.164:6809/395559117]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]} : dispatch 2024-09-10T03:37:28.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 239042 10.00 MB/sec execute 517 sec latency 745.012 ms 2024-09-10T03:37:28.489 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:28 smithi130 ceph-mon[63591]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-10T03:37:28.489 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:28 smithi130 ceph-mon[63591]: osd.5 [v2:172.21.15.164:6808/395559117,v1:172.21.15.164:6809/395559117] boot 2024-09-10T03:37:28.489 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:28 smithi130 ceph-mon[63591]: osdmap e135: 8 total, 8 up, 8 in 2024-09-10T03:37:28.489 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:28 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-10T03:37:28.489 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:28 smithi130 ceph-mon[63591]: osdmap e136: 8 total, 8 up, 8 in 2024-09-10T03:37:28.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:28 smithi164 ceph-mon[49318]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-10T03:37:28.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:28 smithi164 ceph-mon[49318]: osd.5 [v2:172.21.15.164:6808/395559117,v1:172.21.15.164:6809/395559117] boot 2024-09-10T03:37:28.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:28 smithi164 ceph-mon[49318]: osdmap e135: 8 total, 8 up, 8 in 2024-09-10T03:37:28.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:28 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-10T03:37:28.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:28 smithi164 ceph-mon[49318]: osdmap e136: 8 total, 8 up, 8 in 2024-09-10T03:37:29.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 239242 9.99 MB/sec execute 518 sec latency 1061.511 ms 2024-09-10T03:37:29.529 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:29 smithi130 ceph-mon[63591]: pgmap v244: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.9 MiB/s wr, 442 op/s; 185/1701 objects degraded (10.876%) 2024-09-10T03:37:29.529 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:29 smithi130 ceph-mon[63591]: osdmap e137: 8 total, 8 up, 8 in 2024-09-10T03:37:29.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:29 smithi164 ceph-mon[49318]: pgmap v244: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.9 MiB/s wr, 442 op/s; 185/1701 objects degraded (10.876%) 2024-09-10T03:37:29.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:29 smithi164 ceph-mon[49318]: osdmap e137: 8 total, 8 up, 8 in 2024-09-10T03:37:30.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 239514 9.97 MB/sec execute 519 sec latency 972.403 ms 2024-09-10T03:37:31.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 239807 9.97 MB/sec execute 520 sec latency 80.645 ms 2024-09-10T03:37:31.314 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:31 smithi130 ceph-mon[63591]: pgmap v247: 97 pgs: 1 remapped+peering, 6 active+undersized, 22 active+undersized+degraded, 68 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 2.8 MiB/s wr, 259 op/s; 180/1713 objects degraded (10.508%) 2024-09-10T03:37:31.314 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:31 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:31.314 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:31 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:37:31.314 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:31 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 185/1701 objects degraded (10.876%), 23 pgs degraded (PG_DEGRADED) 2024-09-10T03:37:31.314 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:31 smithi130 ceph-mon[63591]: osdmap e138: 8 total, 8 up, 8 in 2024-09-10T03:37:31.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:31 smithi164 ceph-mon[49318]: pgmap v247: 97 pgs: 1 remapped+peering, 6 active+undersized, 22 active+undersized+degraded, 68 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 2.8 MiB/s wr, 259 op/s; 180/1713 objects degraded (10.508%) 2024-09-10T03:37:31.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:31 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:31.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:31 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:37:31.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:31 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 185/1701 objects degraded (10.876%), 23 pgs degraded (PG_DEGRADED) 2024-09-10T03:37:31.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:31 smithi164 ceph-mon[49318]: osdmap e138: 8 total, 8 up, 8 in 2024-09-10T03:37:32.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 240147 9.96 MB/sec execute 521 sec latency 40.323 ms 2024-09-10T03:37:32.338 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:32 smithi130 ceph-mon[63591]: osdmap e139: 8 total, 8 up, 8 in 2024-09-10T03:37:32.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:32 smithi164 ceph-mon[49318]: osdmap e139: 8 total, 8 up, 8 in 2024-09-10T03:37:33.121 INFO:teuthology.orchestra.run.smithi130.stdout: 5 240533 9.96 MB/sec execute 522 sec latency 56.997 ms 2024-09-10T03:37:33.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:33 smithi130 ceph-mon[63591]: pgmap v250: 97 pgs: 2 active+undersized+degraded, 95 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 3.9 MiB/s wr, 487 op/s; 15/1581 objects degraded (0.949%); 1.7 MiB/s, 47 objects/s recovering 2024-09-10T03:37:33.358 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:33 smithi130 ceph-mon[63591]: osdmap e140: 8 total, 8 up, 8 in 2024-09-10T03:37:33.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:33 smithi164 ceph-mon[49318]: pgmap v250: 97 pgs: 2 active+undersized+degraded, 95 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 3.9 MiB/s wr, 487 op/s; 15/1581 objects degraded (0.949%); 1.7 MiB/s, 47 objects/s recovering 2024-09-10T03:37:33.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:33 smithi164 ceph-mon[49318]: osdmap e140: 8 total, 8 up, 8 in 2024-09-10T03:37:34.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 240896 9.95 MB/sec execute 523 sec latency 41.404 ms 2024-09-10T03:37:34.371 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:34 smithi130 ceph-mon[63591]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/1581 objects degraded (0.949%), 2 pgs degraded) 2024-09-10T03:37:34.371 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:34 smithi130 ceph-mon[63591]: Cluster is now healthy 2024-09-10T03:37:34.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:34 smithi164 ceph-mon[49318]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/1581 objects degraded (0.949%), 2 pgs degraded) 2024-09-10T03:37:34.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:34 smithi164 ceph-mon[49318]: Cluster is now healthy 2024-09-10T03:37:35.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 241167 9.96 MB/sec execute 524 sec latency 39.114 ms 2024-09-10T03:37:35.405 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:35 smithi130 ceph-mon[63591]: pgmap v252: 97 pgs: 1 peering, 96 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 3.5 MiB/s wr, 455 op/s; 1.7 MiB/s, 47 objects/s recovering 2024-09-10T03:37:35.609 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:35 smithi164 ceph-mon[49318]: pgmap v252: 97 pgs: 1 peering, 96 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 3.5 MiB/s wr, 455 op/s; 1.7 MiB/s, 47 objects/s recovering 2024-09-10T03:37:36.120 INFO:teuthology.orchestra.run.smithi130.stdout: 5 241491 9.96 MB/sec execute 525 sec latency 67.796 ms 2024-09-10T03:37:37.121 INFO:teuthology.orchestra.run.smithi130.stdout: 5 241775 9.97 MB/sec execute 526 sec latency 59.866 ms 2024-09-10T03:37:37.435 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:37 smithi130 ceph-mon[63591]: pgmap v253: 97 pgs: 1 peering, 96 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.4 MiB/s wr, 435 op/s; 1.3 MiB/s, 35 objects/s recovering 2024-09-10T03:37:37.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:37 smithi164 ceph-mon[49318]: pgmap v253: 97 pgs: 1 peering, 96 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.4 MiB/s wr, 435 op/s; 1.3 MiB/s, 35 objects/s recovering 2024-09-10T03:37:38.121 INFO:teuthology.orchestra.run.smithi130.stdout: 5 241973 9.96 MB/sec execute 527 sec latency 81.155 ms 2024-09-10T03:37:39.121 INFO:teuthology.orchestra.run.smithi130.stdout: 5 242234 9.96 MB/sec execute 528 sec latency 89.944 ms 2024-09-10T03:37:39.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:39 smithi164 ceph-mon[49318]: pgmap v254: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.1 MiB/s wr, 563 op/s; 1.0 MiB/s, 27 objects/s recovering 2024-09-10T03:37:39.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:39 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-10T03:37:39.559 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:39 smithi130 ceph-mon[63591]: pgmap v254: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.1 MiB/s wr, 563 op/s; 1.0 MiB/s, 27 objects/s recovering 2024-09-10T03:37:39.560 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:39 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-10T03:37:40.121 INFO:teuthology.orchestra.run.smithi130.stdout: 5 242406 9.95 MB/sec execute 529 sec latency 72.777 ms 2024-09-10T03:37:40.344 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:40 smithi130 ceph-mon[63591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-10T03:37:40.344 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:40 smithi130 ceph-mon[63591]: Upgrade: osd.6 is safe to restart 2024-09-10T03:37:40.344 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:40 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:40.344 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:40 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-10T03:37:40.344 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:40 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:37:40.642 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:40 smithi164 ceph-mon[49318]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-10T03:37:40.643 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:40 smithi164 ceph-mon[49318]: Upgrade: osd.6 is safe to restart 2024-09-10T03:37:40.643 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:40 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:40.643 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:40 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-10T03:37:40.643 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:40 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:37:41.121 INFO:teuthology.orchestra.run.smithi130.stdout: 5 242612 9.93 MB/sec execute 530 sec latency 45.833 ms 2024-09-10T03:37:41.370 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:41 smithi130 ceph-mon[63591]: pgmap v255: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.5 MiB/s wr, 308 op/s; 897 KiB/s, 16 objects/s recovering 2024-09-10T03:37:41.370 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:41 smithi130 ceph-mon[63591]: Upgrade: Updating osd.6 2024-09-10T03:37:41.370 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:41 smithi130 ceph-mon[63591]: Deploying daemon osd.6 on smithi164 2024-09-10T03:37:41.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:41 smithi164 ceph-mon[49318]: pgmap v255: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.5 MiB/s wr, 308 op/s; 897 KiB/s, 16 objects/s recovering 2024-09-10T03:37:41.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:41 smithi164 ceph-mon[49318]: Upgrade: Updating osd.6 2024-09-10T03:37:41.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:41 smithi164 ceph-mon[49318]: Deploying daemon osd.6 on smithi164 2024-09-10T03:37:42.121 INFO:teuthology.orchestra.run.smithi130.stdout: 5 242880 9.92 MB/sec execute 531 sec latency 55.792 ms 2024-09-10T03:37:43.121 INFO:teuthology.orchestra.run.smithi130.stdout: 5 243123 9.91 MB/sec execute 532 sec latency 57.504 ms 2024-09-10T03:37:43.404 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:43 smithi130 ceph-mon[63591]: pgmap v256: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 405 op/s; 78 KiB/s, 2 objects/s recovering 2024-09-10T03:37:43.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:43 smithi164 ceph-mon[49318]: pgmap v256: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 405 op/s; 78 KiB/s, 2 objects/s recovering 2024-09-10T03:37:43.924 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:43 smithi164 systemd[1]: Stopping Ceph osd.6 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:37:44.121 INFO:teuthology.orchestra.run.smithi130.stdout: 5 243395 9.91 MB/sec execute 533 sec latency 61.084 ms 2024-09-10T03:37:44.207 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:43 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[35907]: 2024-09-10T03:37:43.921+0000 7fbf640e0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:37:44.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:43 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[35907]: 2024-09-10T03:37:43.921+0000 7fbf640e0700 -1 osd.6 140 *** Got signal Terminated *** 2024-09-10T03:37:44.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:43 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[35907]: 2024-09-10T03:37:43.921+0000 7fbf640e0700 -1 osd.6 140 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-10T03:37:44.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:44 smithi130 ceph-mon[63591]: osd.6 marked itself down and dead 2024-09-10T03:37:44.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:44 smithi164 ceph-mon[49318]: osd.6 marked itself down and dead 2024-09-10T03:37:45.121 INFO:teuthology.orchestra.run.smithi130.stdout: 5 243434 9.89 MB/sec execute 534 sec latency 818.164 ms 2024-09-10T03:37:45.282 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:44 smithi164 podman[56768]: 2024-09-10 03:37:44.960738415 +0000 UTC m=+1.155846418 container died afca6f7361e2f690b27ec2f9d21d85e321dd73ae15a9579c3eb966e13ae9cf62 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6, RELEASE=HEAD, distribution-scope=public, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, vendor=Red Hat, Inc., GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, release=754, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, architecture=x86_64) 2024-09-10T03:37:45.620 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:45 smithi130 ceph-mon[63591]: pgmap v257: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.8 MiB/s wr, 346 op/s 2024-09-10T03:37:45.620 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:45 smithi130 ceph-mon[63591]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-10T03:37:45.620 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:45 smithi130 ceph-mon[63591]: osdmap e141: 8 total, 7 up, 8 in 2024-09-10T03:37:45.620 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:45 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:45.620 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:45 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:37:45.620 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:45 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:45.633 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:45 smithi164 ceph-mon[49318]: pgmap v257: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.8 MiB/s wr, 346 op/s 2024-09-10T03:37:45.633 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:45 smithi164 ceph-mon[49318]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-10T03:37:45.633 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:45 smithi164 ceph-mon[49318]: osdmap e141: 8 total, 7 up, 8 in 2024-09-10T03:37:45.633 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:45 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:45.633 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:45 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:37:45.634 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:45 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:45.634 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:45 smithi164 podman[56768]: 2024-09-10 03:37:45.381329081 +0000 UTC m=+1.576437083 container cleanup afca6f7361e2f690b27ec2f9d21d85e321dd73ae15a9579c3eb966e13ae9cf62 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, name=centos-stream, RELEASE=HEAD, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, version=8, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.openshift.expose-services=, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, vcs-type=git, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux ) 2024-09-10T03:37:45.634 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:45 smithi164 bash[56768]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6 2024-09-10T03:37:45.958 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:45 smithi164 podman[56789]: 2024-09-10 03:37:45.632988678 +0000 UTC m=+0.665198773 container remove afca6f7361e2f690b27ec2f9d21d85e321dd73ae15a9579c3eb966e13ae9cf62 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, ceph=True, release=754, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, name=centos-stream, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , version=8, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, vendor=Red Hat, Inc.) 2024-09-10T03:37:46.121 INFO:teuthology.orchestra.run.smithi130.stdout: 5 243743 9.88 MB/sec execute 535 sec latency 1054.390 ms 2024-09-10T03:37:46.366 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:46 smithi164 podman[56873]: 2024-09-10 03:37:45.989837425 +0000 UTC m=+0.018347224 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:37:46.386 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:46 smithi130 ceph-mon[63591]: osdmap e142: 8 total, 7 up, 8 in 2024-09-10T03:37:46.386 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:46 smithi130 ceph-mon[63591]: pgmap v260: 97 pgs: 8 stale+active+clean, 6 active+undersized+degraded, 83 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 3.7 MiB/s wr, 353 op/s; 40/1554 objects degraded (2.574%) 2024-09-10T03:37:46.639 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:46 smithi164 ceph-mon[49318]: osdmap e142: 8 total, 7 up, 8 in 2024-09-10T03:37:46.640 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:46 smithi164 ceph-mon[49318]: pgmap v260: 97 pgs: 8 stale+active+clean, 6 active+undersized+degraded, 83 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 3.7 MiB/s wr, 353 op/s; 40/1554 objects degraded (2.574%) 2024-09-10T03:37:46.640 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:46 smithi164 podman[56873]: 2024-09-10 03:37:46.384329774 +0000 UTC m=+0.412839599 container create 33b976c6171391d390a0c916ed2748482e1b839a7a561cd54cef87fc83042d4b (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-10T03:37:46.958 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:46 smithi164 podman[56873]: 2024-09-10 03:37:46.639237445 +0000 UTC m=+0.667747233 container init 33b976c6171391d390a0c916ed2748482e1b839a7a561cd54cef87fc83042d4b (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-10T03:37:46.958 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:46 smithi164 podman[56873]: 2024-09-10 03:37:46.642946392 +0000 UTC m=+0.671456183 container start 33b976c6171391d390a0c916ed2748482e1b839a7a561cd54cef87fc83042d4b (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-10T03:37:46.958 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:46 smithi164 podman[56873]: 2024-09-10 03:37:46.772865545 +0000 UTC m=+0.801375347 container attach 33b976c6171391d390a0c916ed2748482e1b839a7a561cd54cef87fc83042d4b (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-deactivate, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-10T03:37:46.958 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:46 smithi164 podman[56873]: 2024-09-10 03:37:46.842832459 +0000 UTC m=+0.871342243 container died 33b976c6171391d390a0c916ed2748482e1b839a7a561cd54cef87fc83042d4b (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:37:47.121 INFO:teuthology.orchestra.run.smithi130.stdout: 5 244179 9.88 MB/sec execute 536 sec latency 65.171 ms 2024-09-10T03:37:47.424 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:47 smithi130 ceph-mon[63591]: Health check failed: Degraded data redundancy: 40/1554 objects degraded (2.574%), 6 pgs degraded (PG_DEGRADED) 2024-09-10T03:37:47.619 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:47 smithi164 ceph-mon[49318]: Health check failed: Degraded data redundancy: 40/1554 objects degraded (2.574%), 6 pgs degraded (PG_DEGRADED) 2024-09-10T03:37:47.958 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:47 smithi164 podman[56896]: 2024-09-10 03:37:47.61927545 +0000 UTC m=+0.769824606 container remove 33b976c6171391d390a0c916ed2748482e1b839a7a561cd54cef87fc83042d4b (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-10T03:37:47.958 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:47 smithi164 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-10T03:37:47.958 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:47 smithi164 systemd[1]: Stopped Ceph osd.6 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:37:47.958 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:47 smithi164 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.6.service: Consumed 4min 13.172s CPU time. 2024-09-10T03:37:47.958 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:47 smithi164 systemd[1]: Starting Ceph osd.6 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:37:48.121 INFO:teuthology.orchestra.run.smithi130.stdout: 5 244566 9.88 MB/sec execute 537 sec latency 52.800 ms 2024-09-10T03:37:48.365 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:48 smithi164 podman[56990]: 2024-09-10 03:37:47.999357581 +0000 UTC m=+0.022958836 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:37:48.366 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:48 smithi164 podman[56990]: 2024-09-10 03:37:48.104035299 +0000 UTC m=+0.127636551 container create 63ce574ef600b3671a92c90ef932587bfaf3730ac2267de76377e7b2346ab2eb (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-activate, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-10T03:37:48.469 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:48 smithi130 ceph-mon[63591]: pgmap v261: 97 pgs: 5 active+undersized, 4 stale+active+clean, 19 active+undersized+degraded, 69 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.2 MiB/s wr, 487 op/s; 141/1506 objects degraded (9.363%) 2024-09-10T03:37:48.640 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:37:48.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:48 smithi164 ceph-mon[49318]: pgmap v261: 97 pgs: 5 active+undersized, 4 stale+active+clean, 19 active+undersized+degraded, 69 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.2 MiB/s wr, 487 op/s; 141/1506 objects degraded (9.363%) 2024-09-10T03:37:48.708 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:48 smithi164 podman[56990]: 2024-09-10 03:37:48.365233494 +0000 UTC m=+0.388834760 container init 63ce574ef600b3671a92c90ef932587bfaf3730ac2267de76377e7b2346ab2eb (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-activate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:37:48.708 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:48 smithi164 podman[56990]: 2024-09-10 03:37:48.370047444 +0000 UTC m=+0.393648688 container start 63ce574ef600b3671a92c90ef932587bfaf3730ac2267de76377e7b2346ab2eb (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-10T03:37:48.708 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:48 smithi164 podman[56990]: 2024-09-10 03:37:48.465597987 +0000 UTC m=+0.489199244 container attach 63ce574ef600b3671a92c90ef932587bfaf3730ac2267de76377e7b2346ab2eb (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-activate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_BRANCH=HEAD) 2024-09-10T03:37:48.708 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:48 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-activate[57005]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:37:48.708 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:48 smithi164 bash[56990]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:37:48.709 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:48 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-activate[57005]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:37:48.709 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:48 smithi164 bash[56990]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:37:49.121 INFO:teuthology.orchestra.run.smithi130.stdout: 5 244797 9.88 MB/sec execute 538 sec latency 61.225 ms 2024-09-10T03:37:49.489 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:49 smithi130 ceph-mon[63591]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:37:49.489 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:49 smithi130 ceph-mon[63591]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:37:49.575 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:49 smithi164 ceph-mon[49318]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:37:49.576 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:49 smithi164 ceph-mon[49318]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:37:49.576 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:49 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-activate[57005]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-10T03:37:49.576 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:49 smithi164 bash[56990]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-10T03:37:49.576 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:49 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-activate[57005]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-10T03:37:49.576 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:49 smithi164 bash[56990]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-10T03:37:49.837 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:37:49.837 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (8m) 105s ago 17m 19.4M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:37:49.837 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (5m) 105s ago 17m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 8c7cfb7e87a1 2024-09-10T03:37:49.837 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (5m) 31s ago 16m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 243c094648b8 2024-09-10T03:37:49.837 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (8m) 105s ago 17m 55.9M - dad864ee21e9 72ba01ac1871 2024-09-10T03:37:49.837 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (11m) 105s ago 11m 37.4M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:37:49.837 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (11m) 31s ago 11m 16.7M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:37:49.837 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (7m) 105s ago 18m 519M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:37:49.837 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283,8765 running (6m) 31s ago 16m 457M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e4034e8e2a6d 2024-09-10T03:37:49.838 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (6m) 105s ago 18m 53.2M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 22e5cf56527a 2024-09-10T03:37:49.838 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (6m) 31s ago 16m 44.7M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 7d80bd0d15b9 2024-09-10T03:37:49.838 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (11m) 105s ago 11m 284M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:37:49.838 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (11m) 31s ago 11m 87.9M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:37:49.838 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (8m) 105s ago 17m 10.7M - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:37:49.838 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (7m) 31s ago 16m 11.0M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:37:49.838 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (4m) 105s ago 16m 353M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b de3cbff0098b 2024-09-10T03:37:49.838 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (3m) 105s ago 15m 273M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f0722a6d1c90 2024-09-10T03:37:49.838 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (2m) 105s ago 15m 209M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b a871d2ccf96e 2024-09-10T03:37:49.838 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (115s) 105s ago 14m 14.2M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f2bc83e91fa2 2024-09-10T03:37:49.838 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (72s) 31s ago 14m 262M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 5e1e1b5e1266 2024-09-10T03:37:49.838 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (35s) 31s ago 13m 11.2M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 76ab1aa88b44 2024-09-10T03:37:49.838 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (13m) 31s ago 13m 481M 2704M 17.2.0 e1d6a67b021e afca6f7361e2 2024-09-10T03:37:49.838 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (13m) 31s ago 13m 786M 2704M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:37:49.839 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (6m) 105s ago 16m 54.5M - 2.51.0 1d3b7f56885b f59b02fac5b3 2024-09-10T03:37:49.945 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:49 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-activate[57005]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-10T03:37:49.945 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:49 smithi164 bash[56990]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-10T03:37:49.945 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:49 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-activate[57005]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-10T03:37:49.945 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:49 smithi164 bash[56990]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-10T03:37:49.945 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:49 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-activate[57005]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-10T03:37:49.945 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:49 smithi164 bash[56990]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-10T03:37:49.945 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:49 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-activate[57005]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-10T03:37:49.945 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:49 smithi164 bash[56990]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-10T03:37:49.946 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:49 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-activate[57005]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-10T03:37:49.946 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:49 smithi164 bash[56990]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-10T03:37:49.946 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:49 smithi164 podman[56990]: 2024-09-10 03:37:49.606162572 +0000 UTC m=+1.629763825 container died 63ce574ef600b3671a92c90ef932587bfaf3730ac2267de76377e7b2346ab2eb (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-activate, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-10T03:37:50.121 INFO:teuthology.orchestra.run.smithi130.stdout: 5 245094 9.89 MB/sec execute 539 sec latency 58.030 ms 2024-09-10T03:37:50.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:50 smithi130 ceph-mon[63591]: pgmap v262: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 2.6 MiB/s wr, 303 op/s; 181/1521 objects degraded (11.900%) 2024-09-10T03:37:50.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:50 smithi130 ceph-mon[63591]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:37:50.518 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:37:50.518 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:37:50.518 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:37:50.518 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:37:50.518 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:37:50.518 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:37:50.518 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:37:50.518 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:37:50.518 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-10T03:37:50.518 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 6 2024-09-10T03:37:50.518 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:37:50.519 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:37:50.519 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:37:50.519 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:37:50.519 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:37:50.519 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-10T03:37:50.519 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 10 2024-09-10T03:37:50.519 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:37:50.519 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:37:50.694 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:50 smithi164 ceph-mon[49318]: pgmap v262: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 2.6 MiB/s wr, 303 op/s; 181/1521 objects degraded (11.900%) 2024-09-10T03:37:50.695 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:50 smithi164 ceph-mon[49318]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:37:50.695 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:50 smithi164 podman[57145]: 2024-09-10 03:37:50.337515121 +0000 UTC m=+0.722003790 container remove 63ce574ef600b3671a92c90ef932587bfaf3730ac2267de76377e7b2346ab2eb (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-10T03:37:50.695 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:50 smithi164 podman[57205]: 2024-09-10 03:37:50.449154736 +0000 UTC m=+0.016647342 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:37:50.695 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:50 smithi164 podman[57205]: 2024-09-10 03:37:50.580185458 +0000 UTC m=+0.147678054 container create ca90894c3878ac1695d1e9a5dbeb2a0b43bacc49c16a889a166873930799dec9 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-10T03:37:51.074 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:37:51.074 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:37:51.074 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:37:51.074 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:37:51.074 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-09-10T03:37:51.074 INFO:teuthology.orchestra.run.smithi130.stdout: "mon", 2024-09-10T03:37:51.074 INFO:teuthology.orchestra.run.smithi130.stdout: "crash", 2024-09-10T03:37:51.074 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-09-10T03:37:51.074 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-09-10T03:37:51.075 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "12/23 daemons upgraded", 2024-09-10T03:37:51.075 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading osd daemons", 2024-09-10T03:37:51.075 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:37:51.075 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:37:51.121 INFO:teuthology.orchestra.run.smithi130.stdout: 5 245369 9.89 MB/sec execute 540 sec latency 50.247 ms 2024-09-10T03:37:51.207 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:50 smithi164 podman[57205]: 2024-09-10 03:37:50.824843596 +0000 UTC m=+0.392336193 container init ca90894c3878ac1695d1e9a5dbeb2a0b43bacc49c16a889a166873930799dec9 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-10T03:37:51.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:50 smithi164 podman[57205]: 2024-09-10 03:37:50.828311616 +0000 UTC m=+0.395804214 container start ca90894c3878ac1695d1e9a5dbeb2a0b43bacc49c16a889a166873930799dec9 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:37:51.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:50 smithi164 ceph-osd[57224]: -- 172.21.15.164:0/3366179497 <== mon.1 v2:172.21.15.164:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558522a481e0 con 0x558521be7c00 2024-09-10T03:37:51.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:50 smithi164 bash[57205]: ca90894c3878ac1695d1e9a5dbeb2a0b43bacc49c16a889a166873930799dec9 2024-09-10T03:37:51.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:50 smithi164 systemd[1]: Started Ceph osd.6 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:37:51.556 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:51 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/671395320' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:37:51.556 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:51 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:51.556 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:51 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:51.556 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:51 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:37:51.556 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:51 smithi130 ceph-mon[63591]: from='client.44339 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:37:51.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:51 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/671395320' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:37:51.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:51 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:51.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:51 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:51.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:51 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:37:51.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:51 smithi164 ceph-mon[49318]: from='client.44339 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:37:51.744 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 181/1521 objects degraded (11.900%), 25 pgs degraded 2024-09-10T03:37:51.745 INFO:teuthology.orchestra.run.smithi130.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-10T03:37:51.745 INFO:teuthology.orchestra.run.smithi130.stdout: osd.6 (root=default,host=smithi164) is down 2024-09-10T03:37:51.745 INFO:teuthology.orchestra.run.smithi130.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 181/1521 objects degraded (11.900%), 25 pgs degraded 2024-09-10T03:37:51.745 INFO:teuthology.orchestra.run.smithi130.stdout: pg 1.0 is active+undersized+degraded, acting [7,0] 2024-09-10T03:37:51.745 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.2 is active+undersized+degraded, acting [5,1] 2024-09-10T03:37:51.745 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.6 is active+undersized+degraded, acting [1,4] 2024-09-10T03:37:51.745 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-09-10T03:37:51.745 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-09-10T03:37:51.745 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.14 is active+undersized+degraded, acting [3,5] 2024-09-10T03:37:51.745 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.16 is active+undersized+degraded, acting [5,2] 2024-09-10T03:37:51.745 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-09-10T03:37:51.746 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.19 is active+undersized+degraded, acting [3,0] 2024-09-10T03:37:51.746 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.1a is active+undersized+degraded, acting [4,7] 2024-09-10T03:37:51.746 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.1b is active+undersized+degraded, acting [3,7] 2024-09-10T03:37:51.746 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.1d is active+undersized+degraded, acting [7,0] 2024-09-10T03:37:51.746 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-09-10T03:37:51.746 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.2 is active+undersized+degraded, acting [3,5] 2024-09-10T03:37:51.746 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-09-10T03:37:51.746 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.a is active+undersized+degraded, acting [4,1] 2024-09-10T03:37:51.746 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.c is active+undersized+degraded, acting [5,3] 2024-09-10T03:37:51.746 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.d is active+undersized+degraded, acting [7,5] 2024-09-10T03:37:51.746 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-09-10T03:37:51.746 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.11 is active+undersized+degraded, acting [7,4] 2024-09-10T03:37:51.746 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-09-10T03:37:51.746 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-09-10T03:37:51.746 INFO:teuthology.orchestra.run.smithi130.stdout: pg 3.1e is active+undersized+degraded, acting [3,2] 2024-09-10T03:37:51.747 INFO:teuthology.orchestra.run.smithi130.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-09-10T03:37:51.747 INFO:teuthology.orchestra.run.smithi130.stdout: pg 4.a is active+undersized+degraded, acting [1,7] 2024-09-10T03:37:52.121 INFO:teuthology.orchestra.run.smithi130.stdout: 5 245623 9.89 MB/sec execute 541 sec latency 47.710 ms 2024-09-10T03:37:52.590 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:52 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/554686740' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:37:52.590 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:52 smithi130 ceph-mon[63591]: pgmap v263: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 450 op/s; 194/1632 objects degraded (11.887%) 2024-09-10T03:37:52.630 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:52 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/554686740' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:37:52.630 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:52 smithi164 ceph-mon[49318]: pgmap v263: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 450 op/s; 194/1632 objects degraded (11.887%) 2024-09-10T03:37:52.957 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:37:52 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:37:52.722+0000 7f1a358f2640 -1 Falling back to public interface 2024-09-10T03:37:53.122 INFO:teuthology.orchestra.run.smithi130.stdout: 5 245845 9.88 MB/sec execute 542 sec latency 68.924 ms 2024-09-10T03:37:53.619 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:53 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 194/1632 objects degraded (11.887%), 25 pgs degraded (PG_DEGRADED) 2024-09-10T03:37:53.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:53 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 194/1632 objects degraded (11.887%), 25 pgs degraded (PG_DEGRADED) 2024-09-10T03:37:54.126 INFO:teuthology.orchestra.run.smithi130.stdout: 5 246024 9.87 MB/sec execute 543 sec latency 54.652 ms 2024-09-10T03:37:54.632 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:54 smithi130 ceph-mon[63591]: pgmap v264: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.0 MiB/s wr, 428 op/s; 194/1656 objects degraded (11.715%) 2024-09-10T03:37:54.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:54 smithi164 ceph-mon[49318]: pgmap v264: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.0 MiB/s wr, 428 op/s; 194/1656 objects degraded (11.715%) 2024-09-10T03:37:55.126 INFO:teuthology.orchestra.run.smithi130.stdout: 5 246204 9.85 MB/sec execute 544 sec latency 88.479 ms 2024-09-10T03:37:56.126 INFO:teuthology.orchestra.run.smithi130.stdout: 5 246452 9.84 MB/sec execute 545 sec latency 53.321 ms 2024-09-10T03:37:56.166 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:55 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:56.166 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:55 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:56.166 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:55 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:56.166 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:55 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:56.305 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:55 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:56.305 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:55 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:56.305 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:55 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:56.305 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:55 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:37:57.002 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:56 smithi130 ceph-mon[63591]: pgmap v265: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.4 MiB/s wr, 415 op/s; 199/1677 objects degraded (11.866%) 2024-09-10T03:37:57.126 INFO:teuthology.orchestra.run.smithi130.stdout: 5 246767 9.84 MB/sec execute 546 sec latency 63.624 ms 2024-09-10T03:37:57.337 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:56 smithi164 ceph-mon[49318]: pgmap v265: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.4 MiB/s wr, 415 op/s; 199/1677 objects degraded (11.866%) 2024-09-10T03:37:58.042 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:58 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 199/1641 objects degraded (12.127%), 25 pgs degraded (PG_DEGRADED) 2024-09-10T03:37:58.126 INFO:teuthology.orchestra.run.smithi130.stdout: 5 247044 9.83 MB/sec execute 547 sec latency 75.406 ms 2024-09-10T03:37:58.360 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:58 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 199/1641 objects degraded (12.127%), 25 pgs degraded (PG_DEGRADED) 2024-09-10T03:37:59.056 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:37:59 smithi130 ceph-mon[63591]: pgmap v266: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.0 MiB/s wr, 399 op/s; 199/1641 objects degraded (12.127%) 2024-09-10T03:37:59.126 INFO:teuthology.orchestra.run.smithi130.stdout: 5 247385 9.83 MB/sec execute 548 sec latency 84.568 ms 2024-09-10T03:37:59.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:37:59 smithi164 ceph-mon[49318]: pgmap v266: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.0 MiB/s wr, 399 op/s; 199/1641 objects degraded (12.127%) 2024-09-10T03:38:00.126 INFO:teuthology.orchestra.run.smithi130.stdout: 5 247736 9.82 MB/sec execute 549 sec latency 60.586 ms 2024-09-10T03:38:00.365 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:00.365 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:00.365 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:38:00.365 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:38:00.365 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:00.365 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:00.365 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:38:00.365 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:38:00.365 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:38:00.366 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:38:00.366 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-10T03:38:00.366 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:00.366 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:38:00.457 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:38:00 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:38:00.179+0000 7f1a358f2640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-10T03:38:00.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:00.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:00.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:38:00.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:38:00.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:00.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:00.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:38:00.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:38:00.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:38:00.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:38:00.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-10T03:38:00.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:00.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:38:01.126 INFO:teuthology.orchestra.run.smithi130.stdout: 5 248044 9.82 MB/sec execute 550 sec latency 103.642 ms 2024-09-10T03:38:01.378 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:01 smithi130 ceph-mon[63591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-10T03:38:01.378 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:01 smithi130 ceph-mon[63591]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-10T03:38:01.378 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:01 smithi130 ceph-mon[63591]: pgmap v267: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.9 MiB/s wr, 389 op/s; 199/1635 objects degraded (12.171%) 2024-09-10T03:38:01.457 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:38:01 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:38:01.331+0000 7f1a358f2640 -1 osd.6 140 log_to_monitors true 2024-09-10T03:38:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:01 smithi164 ceph-mon[49318]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-10T03:38:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:01 smithi164 ceph-mon[49318]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-10T03:38:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:01 smithi164 ceph-mon[49318]: pgmap v267: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.9 MiB/s wr, 389 op/s; 199/1635 objects degraded (12.171%) 2024-09-10T03:38:02.126 INFO:teuthology.orchestra.run.smithi130.stdout: 5 248314 9.82 MB/sec execute 551 sec latency 41.587 ms 2024-09-10T03:38:02.169 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:02 smithi130 ceph-mon[63591]: from='osd.6 [v2:172.21.15.164:6816/2378902145,v1:172.21.15.164:6817/2378902145]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-10T03:38:02.169 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:02 smithi130 ceph-mon[63591]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-10T03:38:02.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:02 smithi164 ceph-mon[49318]: from='osd.6 [v2:172.21.15.164:6816/2378902145,v1:172.21.15.164:6817/2378902145]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-10T03:38:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:02 smithi164 ceph-mon[49318]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-10T03:38:02.458 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:38:02 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:38:02.168+0000 7f1a2d69c640 -1 osd.6 140 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-10T03:38:03.126 INFO:teuthology.orchestra.run.smithi130.stdout: 5 248642 9.83 MB/sec execute 552 sec latency 47.123 ms 2024-09-10T03:38:03.181 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:03 smithi130 ceph-mon[63591]: pgmap v268: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.3 MiB/s wr, 447 op/s; 189/1569 objects degraded (12.046%) 2024-09-10T03:38:03.181 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:03 smithi130 ceph-mon[63591]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-10T03:38:03.181 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:03 smithi130 ceph-mon[63591]: from='osd.6 [v2:172.21.15.164:6816/2378902145,v1:172.21.15.164:6817/2378902145]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]} : dispatch 2024-09-10T03:38:03.181 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:03 smithi130 ceph-mon[63591]: osdmap e143: 8 total, 7 up, 8 in 2024-09-10T03:38:03.181 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:03 smithi130 ceph-mon[63591]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]} : dispatch 2024-09-10T03:38:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:03 smithi164 ceph-mon[49318]: pgmap v268: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.3 MiB/s wr, 447 op/s; 189/1569 objects degraded (12.046%) 2024-09-10T03:38:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:03 smithi164 ceph-mon[49318]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-10T03:38:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:03 smithi164 ceph-mon[49318]: from='osd.6 [v2:172.21.15.164:6816/2378902145,v1:172.21.15.164:6817/2378902145]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]} : dispatch 2024-09-10T03:38:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:03 smithi164 ceph-mon[49318]: osdmap e143: 8 total, 7 up, 8 in 2024-09-10T03:38:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:03 smithi164 ceph-mon[49318]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]} : dispatch 2024-09-10T03:38:04.126 INFO:teuthology.orchestra.run.smithi130.stdout: 5 248806 9.82 MB/sec execute 553 sec latency 886.738 ms 2024-09-10T03:38:04.217 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:04 smithi130 ceph-mon[63591]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-10T03:38:04.217 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:04 smithi130 ceph-mon[63591]: osd.6 [v2:172.21.15.164:6816/2378902145,v1:172.21.15.164:6817/2378902145] boot 2024-09-10T03:38:04.217 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:04 smithi130 ceph-mon[63591]: osdmap e144: 8 total, 8 up, 8 in 2024-09-10T03:38:04.217 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:04 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-10T03:38:04.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:04 smithi164 ceph-mon[49318]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-10T03:38:04.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:04 smithi164 ceph-mon[49318]: osd.6 [v2:172.21.15.164:6816/2378902145,v1:172.21.15.164:6817/2378902145] boot 2024-09-10T03:38:04.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:04 smithi164 ceph-mon[49318]: osdmap e144: 8 total, 8 up, 8 in 2024-09-10T03:38:04.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:04 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-10T03:38:05.126 INFO:teuthology.orchestra.run.smithi130.stdout: 5 248846 9.80 MB/sec execute 554 sec latency 1886.770 ms 2024-09-10T03:38:05.422 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:05 smithi130 ceph-mon[63591]: pgmap v271: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 3.1 MiB/s wr, 431 op/s; 185/1575 objects degraded (11.746%) 2024-09-10T03:38:05.422 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:05 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 185/1575 objects degraded (11.746%), 25 pgs degraded (PG_DEGRADED) 2024-09-10T03:38:05.422 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:05 smithi130 ceph-mon[63591]: osdmap e145: 8 total, 8 up, 8 in 2024-09-10T03:38:05.422 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:05 smithi130 ceph-mon[63591]: osdmap e146: 8 total, 8 up, 8 in 2024-09-10T03:38:05.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:05 smithi164 ceph-mon[49318]: pgmap v271: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 3.1 MiB/s wr, 431 op/s; 185/1575 objects degraded (11.746%) 2024-09-10T03:38:05.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:05 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 185/1575 objects degraded (11.746%), 25 pgs degraded (PG_DEGRADED) 2024-09-10T03:38:05.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:05 smithi164 ceph-mon[49318]: osdmap e145: 8 total, 8 up, 8 in 2024-09-10T03:38:05.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:05 smithi164 ceph-mon[49318]: osdmap e146: 8 total, 8 up, 8 in 2024-09-10T03:38:06.126 INFO:teuthology.orchestra.run.smithi130.stdout: 5 249068 9.80 MB/sec execute 555 sec latency 2124.228 ms 2024-09-10T03:38:07.126 INFO:teuthology.orchestra.run.smithi130.stdout: 5 249224 9.78 MB/sec execute 556 sec latency 908.113 ms 2024-09-10T03:38:07.436 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:07 smithi130 ceph-mon[63591]: pgmap v274: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 activating, 1 activating+degraded, 1 remapped+peering, 5 active+undersized, 19 active+undersized+degraded, 66 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s wr, 121 op/s; 178/1629 objects degraded (10.927%); 0 B/s, 3 objects/s recovering 2024-09-10T03:38:07.436 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:07 smithi130 ceph-mon[63591]: osdmap e147: 8 total, 8 up, 8 in 2024-09-10T03:38:07.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:07 smithi164 ceph-mon[49318]: pgmap v274: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 activating, 1 activating+degraded, 1 remapped+peering, 5 active+undersized, 19 active+undersized+degraded, 66 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s wr, 121 op/s; 178/1629 objects degraded (10.927%); 0 B/s, 3 objects/s recovering 2024-09-10T03:38:07.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:07 smithi164 ceph-mon[49318]: osdmap e147: 8 total, 8 up, 8 in 2024-09-10T03:38:08.126 INFO:teuthology.orchestra.run.smithi130.stdout: 5 249495 9.79 MB/sec execute 557 sec latency 985.981 ms 2024-09-10T03:38:08.212 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:08 smithi130 ceph-mon[63591]: osdmap e148: 8 total, 8 up, 8 in 2024-09-10T03:38:08.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:08 smithi164 ceph-mon[49318]: osdmap e148: 8 total, 8 up, 8 in 2024-09-10T03:38:09.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 249728 9.78 MB/sec execute 558 sec latency 65.451 ms 2024-09-10T03:38:09.225 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:09 smithi130 ceph-mon[63591]: pgmap v277: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 peering, 4 activating, 1 activating+degraded, 1 remapped+peering, 83 active+clean; 345 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.7 MiB/s wr, 477 op/s; 45/1626 objects degraded (2.768%); 699 KiB/s, 73 keys/s, 33 objects/s recovering 2024-09-10T03:38:09.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:09 smithi164 ceph-mon[49318]: pgmap v277: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 peering, 4 activating, 1 activating+degraded, 1 remapped+peering, 83 active+clean; 345 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.7 MiB/s wr, 477 op/s; 45/1626 objects degraded (2.768%); 699 KiB/s, 73 keys/s, 33 objects/s recovering 2024-09-10T03:38:10.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 250009 9.77 MB/sec execute 559 sec latency 41.879 ms 2024-09-10T03:38:10.258 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:10 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 28/1614 objects degraded (1.735%), 1 pg degraded (PG_DEGRADED) 2024-09-10T03:38:10.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:10 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 28/1614 objects degraded (1.735%), 1 pg degraded (PG_DEGRADED) 2024-09-10T03:38:11.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 250400 9.76 MB/sec execute 560 sec latency 42.933 ms 2024-09-10T03:38:11.280 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:11 smithi130 ceph-mon[63591]: pgmap v278: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 peering, 89 active+clean; 343 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.3 MiB/s wr, 351 op/s; 28/1614 objects degraded (1.735%); 613 KiB/s, 52 keys/s, 29 objects/s recovering 2024-09-10T03:38:11.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:11 smithi164 ceph-mon[49318]: pgmap v278: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 peering, 89 active+clean; 343 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.3 MiB/s wr, 351 op/s; 28/1614 objects degraded (1.735%); 613 KiB/s, 52 keys/s, 29 objects/s recovering 2024-09-10T03:38:12.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 250759 9.76 MB/sec execute 561 sec latency 37.664 ms 2024-09-10T03:38:12.304 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:12 smithi130 ceph-mon[63591]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/1614 objects degraded (1.735%), 1 pg degraded) 2024-09-10T03:38:12.304 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:12 smithi130 ceph-mon[63591]: Cluster is now healthy 2024-09-10T03:38:12.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:12 smithi164 ceph-mon[49318]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/1614 objects degraded (1.735%), 1 pg degraded) 2024-09-10T03:38:12.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:12 smithi164 ceph-mon[49318]: Cluster is now healthy 2024-09-10T03:38:13.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 251194 9.78 MB/sec execute 562 sec latency 41.046 ms 2024-09-10T03:38:13.321 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:13 smithi130 ceph-mon[63591]: pgmap v279: 97 pgs: 4 peering, 93 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.9 MiB/s wr, 485 op/s; 521 KiB/s, 44 keys/s, 29 objects/s recovering 2024-09-10T03:38:13.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:13 smithi164 ceph-mon[49318]: pgmap v279: 97 pgs: 4 peering, 93 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.9 MiB/s wr, 485 op/s; 521 KiB/s, 44 keys/s, 29 objects/s recovering 2024-09-10T03:38:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 251723 9.77 MB/sec execute 563 sec latency 41.952 ms 2024-09-10T03:38:15.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 252083 9.78 MB/sec execute 564 sec latency 43.667 ms 2024-09-10T03:38:15.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:15 smithi130 ceph-mon[63591]: pgmap v280: 97 pgs: 97 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.2 MiB/s wr, 503 op/s; 1.3 MiB/s, 36 keys/s, 26 objects/s recovering 2024-09-10T03:38:15.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:15 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-10T03:38:15.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:15 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:15.361 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:15 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:38:15.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:15 smithi164 ceph-mon[49318]: pgmap v280: 97 pgs: 97 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.2 MiB/s wr, 503 op/s; 1.3 MiB/s, 36 keys/s, 26 objects/s recovering 2024-09-10T03:38:15.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:15 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-10T03:38:15.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:15 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:15.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:15 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:38:16.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 252543 9.79 MB/sec execute 565 sec latency 40.357 ms 2024-09-10T03:38:16.374 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:16 smithi130 ceph-mon[63591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-10T03:38:16.374 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:16 smithi130 ceph-mon[63591]: Upgrade: osd.7 is safe to restart 2024-09-10T03:38:16.374 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:16 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:16.374 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:16 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-10T03:38:16.374 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:16 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:38:16.650 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:16 smithi164 ceph-mon[49318]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-10T03:38:16.650 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:16 smithi164 ceph-mon[49318]: Upgrade: osd.7 is safe to restart 2024-09-10T03:38:16.650 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:16 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:16.650 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:16 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-10T03:38:16.650 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:16 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:38:17.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 253172 9.81 MB/sec execute 566 sec latency 52.101 ms 2024-09-10T03:38:17.385 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:17 smithi130 ceph-mon[63591]: pgmap v281: 97 pgs: 97 active+clean; 351 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.2 MiB/s wr, 573 op/s; 1.1 MiB/s, 30 keys/s, 21 objects/s recovering 2024-09-10T03:38:17.385 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:17 smithi130 ceph-mon[63591]: Upgrade: Updating osd.7 2024-09-10T03:38:17.385 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:17 smithi130 ceph-mon[63591]: Deploying daemon osd.7 on smithi164 2024-09-10T03:38:17.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:17 smithi164 ceph-mon[49318]: pgmap v281: 97 pgs: 97 active+clean; 351 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.2 MiB/s wr, 573 op/s; 1.1 MiB/s, 30 keys/s, 21 objects/s recovering 2024-09-10T03:38:17.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:17 smithi164 ceph-mon[49318]: Upgrade: Updating osd.7 2024-09-10T03:38:17.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:17 smithi164 ceph-mon[49318]: Deploying daemon osd.7 on smithi164 2024-09-10T03:38:18.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 253593 9.80 MB/sec execute 567 sec latency 72.159 ms 2024-09-10T03:38:19.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 253901 9.79 MB/sec execute 568 sec latency 73.971 ms 2024-09-10T03:38:19.435 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:19 smithi130 ceph-mon[63591]: pgmap v282: 97 pgs: 97 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 461 op/s; 774 KiB/s, 8 objects/s recovering 2024-09-10T03:38:19.535 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:19 smithi164 ceph-mon[49318]: pgmap v282: 97 pgs: 97 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 461 op/s; 774 KiB/s, 8 objects/s recovering 2024-09-10T03:38:20.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 254211 9.79 MB/sec execute 569 sec latency 51.213 ms 2024-09-10T03:38:20.356 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:20 smithi164 systemd[1]: Stopping Ceph osd.7 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:38:21.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 254499 9.79 MB/sec execute 570 sec latency 55.079 ms 2024-09-10T03:38:21.219 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:21 smithi130 ceph-mon[63591]: pgmap v283: 97 pgs: 97 active+clean; 349 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.8 MiB/s wr, 502 op/s; 685 KiB/s, 7 objects/s recovering 2024-09-10T03:38:21.219 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:21 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:21.219 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:21 smithi130 ceph-mon[63591]: osd.7 marked itself down and dead 2024-09-10T03:38:21.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:21 smithi164 ceph-mon[49318]: pgmap v283: 97 pgs: 97 active+clean; 349 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.8 MiB/s wr, 502 op/s; 685 KiB/s, 7 objects/s recovering 2024-09-10T03:38:21.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:21 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:21.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:21 smithi164 ceph-mon[49318]: osd.7 marked itself down and dead 2024-09-10T03:38:21.458 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:21 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[38889]: 2024-09-10T03:38:21.143+0000 7efc793e7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:38:21.458 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:21 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[38889]: 2024-09-10T03:38:21.143+0000 7efc793e7700 -1 osd.7 148 *** Got signal Terminated *** 2024-09-10T03:38:21.458 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:21 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[38889]: 2024-09-10T03:38:21.143+0000 7efc793e7700 -1 osd.7 148 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-10T03:38:22.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 254649 9.78 MB/sec execute 571 sec latency 787.953 ms 2024-09-10T03:38:22.129 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:38:22.279 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:22 smithi164 ceph-mon[49318]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-10T03:38:22.279 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:22 smithi164 ceph-mon[49318]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-10T03:38:22.279 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:22 smithi164 ceph-mon[49318]: osdmap e149: 8 total, 7 up, 8 in 2024-09-10T03:38:22.279 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:21 smithi164 podman[58798]: 2024-09-10 03:38:21.996574034 +0000 UTC m=+1.700848340 container died 2e92b3fb269ec87dc0995f05a729b5886286b93171ef393b17b9e906fa4e02c5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7, architecture=x86_64, io.openshift.tags=base centos centos-stream, ceph=True, version=8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, RELEASE=HEAD, GIT_BRANCH=HEAD, io.openshift.expose-services=, name=centos-stream, GIT_CLEAN=True, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-10T03:38:22.380 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:22 smithi130 ceph-mon[63591]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-10T03:38:22.380 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:22 smithi130 ceph-mon[63591]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-10T03:38:22.380 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:22 smithi130 ceph-mon[63591]: osdmap e149: 8 total, 7 up, 8 in 2024-09-10T03:38:22.655 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:22 smithi164 podman[58798]: 2024-09-10 03:38:22.37797397 +0000 UTC m=+2.082248268 container cleanup 2e92b3fb269ec87dc0995f05a729b5886286b93171ef393b17b9e906fa4e02c5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7, maintainer=Guillaume Abrioux , release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, name=centos-stream, GIT_BRANCH=HEAD, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, distribution-scope=public, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, architecture=x86_64, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-10T03:38:22.655 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:22 smithi164 bash[58798]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7 2024-09-10T03:38:22.957 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:22 smithi164 podman[58831]: 2024-09-10 03:38:22.654940248 +0000 UTC m=+0.660861025 container remove 2e92b3fb269ec87dc0995f05a729b5886286b93171ef393b17b9e906fa4e02c5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, distribution-scope=public, ceph=True, GIT_CLEAN=True, io.buildah.version=1.19.8, architecture=x86_64, version=8, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-10T03:38:23.070 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:38:23.070 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (9m) 2m ago 18m 19.4M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:38:23.070 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (5m) 2m ago 18m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 8c7cfb7e87a1 2024-09-10T03:38:23.070 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (5m) 28s ago 17m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 243c094648b8 2024-09-10T03:38:23.070 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (8m) 2m ago 17m 55.9M - dad864ee21e9 72ba01ac1871 2024-09-10T03:38:23.070 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (12m) 2m ago 12m 37.4M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:38:23.070 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (12m) 28s ago 12m 16.9M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:38:23.070 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (8m) 2m ago 19m 519M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:38:23.070 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283,8765 running (7m) 28s ago 17m 457M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e4034e8e2a6d 2024-09-10T03:38:23.071 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (7m) 2m ago 19m 53.2M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 22e5cf56527a 2024-09-10T03:38:23.071 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (6m) 28s ago 17m 45.7M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 7d80bd0d15b9 2024-09-10T03:38:23.071 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (11m) 2m ago 11m 284M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:38:23.071 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (11m) 28s ago 11m 87.9M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:38:23.071 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (8m) 2m ago 17m 10.7M - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:38:23.071 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (8m) 28s ago 17m 10.4M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:38:23.071 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (5m) 2m ago 16m 353M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b de3cbff0098b 2024-09-10T03:38:23.071 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (4m) 2m ago 16m 273M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f0722a6d1c90 2024-09-10T03:38:23.071 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (3m) 2m ago 15m 209M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b a871d2ccf96e 2024-09-10T03:38:23.071 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (2m) 2m ago 15m 14.2M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f2bc83e91fa2 2024-09-10T03:38:23.071 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (105s) 28s ago 14m 296M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 5e1e1b5e1266 2024-09-10T03:38:23.071 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (68s) 28s ago 14m 248M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 76ab1aa88b44 2024-09-10T03:38:23.071 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (32s) 28s ago 13m 14.5M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b ca90894c3878 2024-09-10T03:38:23.072 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (13m) 28s ago 13m 575M 2704M 17.2.0 e1d6a67b021e 2e92b3fb269e 2024-09-10T03:38:23.072 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (7m) 2m ago 17m 54.5M - 2.51.0 1d3b7f56885b f59b02fac5b3 2024-09-10T03:38:23.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 254961 9.78 MB/sec execute 572 sec latency 1022.543 ms 2024-09-10T03:38:23.334 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:23 smithi164 ceph-mon[49318]: pgmap v285: 97 pgs: 12 peering, 10 stale+active+clean, 75 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.3 MiB/s wr, 657 op/s; 756 KiB/s, 2 objects/s recovering 2024-09-10T03:38:23.335 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:23 smithi164 ceph-mon[49318]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:38:23.335 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:23 smithi164 ceph-mon[49318]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-10T03:38:23.335 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:23 smithi164 ceph-mon[49318]: osdmap e150: 8 total, 7 up, 8 in 2024-09-10T03:38:23.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:23 smithi164 podman[58917]: 2024-09-10 03:38:22.930600395 +0000 UTC m=+0.019065285 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:38:23.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:23 smithi164 podman[58917]: 2024-09-10 03:38:23.064365928 +0000 UTC m=+0.152830823 container create 3855ced42c6dace47844bbe4aaae11c94922e2af86d663aba99fff32b2aa7e1b (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-10T03:38:23.401 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:23 smithi130 ceph-mon[63591]: pgmap v285: 97 pgs: 12 peering, 10 stale+active+clean, 75 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.3 MiB/s wr, 657 op/s; 756 KiB/s, 2 objects/s recovering 2024-09-10T03:38:23.401 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:23 smithi130 ceph-mon[63591]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:38:23.401 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:23 smithi130 ceph-mon[63591]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-10T03:38:23.401 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:23 smithi130 ceph-mon[63591]: osdmap e150: 8 total, 7 up, 8 in 2024-09-10T03:38:23.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:23 smithi164 podman[58917]: 2024-09-10 03:38:23.334278361 +0000 UTC m=+0.422743255 container init 3855ced42c6dace47844bbe4aaae11c94922e2af86d663aba99fff32b2aa7e1b (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-deactivate, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:38:23.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:23 smithi164 podman[58917]: 2024-09-10 03:38:23.339082906 +0000 UTC m=+0.427547806 container start 3855ced42c6dace47844bbe4aaae11c94922e2af86d663aba99fff32b2aa7e1b (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-10T03:38:23.709 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:38:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:38:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:38:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:38:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:38:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:38:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:38:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:38:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 7 2024-09-10T03:38:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:38:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:38:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-10T03:38:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:38:23.711 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:38:23.711 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-10T03:38:23.711 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 11 2024-09-10T03:38:23.711 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:38:23.711 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:38:24.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 255375 9.77 MB/sec execute 573 sec latency 38.562 ms 2024-09-10T03:38:24.179 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:23 smithi164 podman[58917]: 2024-09-10 03:38:23.718146367 +0000 UTC m=+0.806611260 container attach 3855ced42c6dace47844bbe4aaae11c94922e2af86d663aba99fff32b2aa7e1b (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-10T03:38:24.179 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:23 smithi164 podman[58917]: 2024-09-10 03:38:23.718443527 +0000 UTC m=+0.806908418 container died 3855ced42c6dace47844bbe4aaae11c94922e2af86d663aba99fff32b2aa7e1b (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-10T03:38:24.251 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:38:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:38:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:38:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:38:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-09-10T03:38:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: "mon", 2024-09-10T03:38:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: "crash", 2024-09-10T03:38:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr" 2024-09-10T03:38:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-09-10T03:38:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "13/23 daemons upgraded", 2024-09-10T03:38:24.252 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading osd daemons", 2024-09-10T03:38:24.252 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:38:24.252 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:38:24.419 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:24 smithi130 ceph-mon[63591]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:38:24.419 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:24 smithi130 ceph-mon[63591]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:38:24.419 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:24 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/1281215685' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:38:24.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:24 smithi164 ceph-mon[49318]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:38:24.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:24 smithi164 ceph-mon[49318]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:38:24.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:24 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/1281215685' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:38:24.458 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:24 smithi164 podman[58940]: 2024-09-10 03:38:24.3401229 +0000 UTC m=+0.749028303 container remove 3855ced42c6dace47844bbe4aaae11c94922e2af86d663aba99fff32b2aa7e1b (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1) 2024-09-10T03:38:24.458 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:24 smithi164 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-10T03:38:24.458 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:24 smithi164 systemd[1]: Stopped Ceph osd.7 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:38:24.458 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:24 smithi164 systemd[1]: ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.7.service: Consumed 6min 47.816s CPU time. 2024-09-10T03:38:24.852 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_WARN 1 osds down; all OSDs are running squid or later but require_osd_release < squid; Reduced data availability: 2 pgs peering 2024-09-10T03:38:24.852 INFO:teuthology.orchestra.run.smithi130.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-10T03:38:24.852 INFO:teuthology.orchestra.run.smithi130.stdout: osd.7 (root=default,host=smithi164) is down 2024-09-10T03:38:24.852 INFO:teuthology.orchestra.run.smithi130.stdout:[WRN] OSD_UPGRADE_FINISHED: all OSDs are running squid or later but require_osd_release < squid 2024-09-10T03:38:24.852 INFO:teuthology.orchestra.run.smithi130.stdout: all OSDs are running squid or later but require_osd_release < squid 2024-09-10T03:38:24.852 INFO:teuthology.orchestra.run.smithi130.stdout:[WRN] PG_AVAILABILITY: Reduced data availability: 2 pgs peering 2024-09-10T03:38:24.852 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.0 is stuck peering for 86s, current state peering, last acting [1,0] 2024-09-10T03:38:24.853 INFO:teuthology.orchestra.run.smithi130.stdout: pg 2.1d is stuck peering for 2m, current state peering, last acting [6,0] 2024-09-10T03:38:25.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 255658 9.78 MB/sec execute 574 sec latency 56.219 ms 2024-09-10T03:38:25.192 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:24 smithi164 systemd[1]: Starting Ceph osd.7 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:38:25.192 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:24 smithi164 podman[59042]: 2024-09-10 03:38:24.87488131 +0000 UTC m=+0.019625889 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:38:25.437 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:25 smithi130 ceph-mon[63591]: pgmap v287: 97 pgs: 25 peering, 8 stale+active+clean, 64 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 5.3 MiB/s wr, 542 op/s 2024-09-10T03:38:25.437 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:25 smithi130 ceph-mon[63591]: from='client.34456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:38:25.437 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:25 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/2086522847' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:38:25.457 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:25 smithi164 podman[59042]: 2024-09-10 03:38:25.191875373 +0000 UTC m=+0.336619958 container create da26e6434d0c003446a2570f73464f14e29303955146cd4b3b35eaf90d2e6a21 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-activate, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:38:25.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:25 smithi164 ceph-mon[49318]: pgmap v287: 97 pgs: 25 peering, 8 stale+active+clean, 64 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 5.3 MiB/s wr, 542 op/s 2024-09-10T03:38:25.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:25 smithi164 ceph-mon[49318]: from='client.34456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:38:25.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:25 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/2086522847' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:38:25.958 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:25 smithi164 podman[59042]: 2024-09-10 03:38:25.48173971 +0000 UTC m=+0.626484290 container init da26e6434d0c003446a2570f73464f14e29303955146cd4b3b35eaf90d2e6a21 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-10T03:38:25.958 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:25 smithi164 podman[59042]: 2024-09-10 03:38:25.486626277 +0000 UTC m=+0.631370855 container start da26e6434d0c003446a2570f73464f14e29303955146cd4b3b35eaf90d2e6a21 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-10T03:38:25.958 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:25 smithi164 podman[59042]: 2024-09-10 03:38:25.588921358 +0000 UTC m=+0.733665964 container attach da26e6434d0c003446a2570f73464f14e29303955146cd4b3b35eaf90d2e6a21 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-activate, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-10T03:38:25.958 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:25 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-activate[59056]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:38:25.958 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:25 smithi164 bash[59042]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:38:25.958 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:25 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-activate[59056]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:38:25.958 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:25 smithi164 bash[59042]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-10T03:38:26.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 256126 9.79 MB/sec execute 575 sec latency 39.370 ms 2024-09-10T03:38:26.435 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:26 smithi164 ceph-mon[49318]: Health check failed: Degraded data redundancy: 140/1587 objects degraded (8.822%), 13 pgs degraded (PG_DEGRADED) 2024-09-10T03:38:26.436 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:26 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-activate[59056]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-10T03:38:26.436 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:26 smithi164 bash[59042]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-10T03:38:26.436 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:26 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-activate[59056]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-10T03:38:26.436 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:26 smithi164 bash[59042]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-10T03:38:26.436 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:26 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-activate[59056]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-10T03:38:26.436 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:26 smithi164 bash[59042]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-10T03:38:26.468 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:26 smithi130 ceph-mon[63591]: Health check failed: Degraded data redundancy: 140/1587 objects degraded (8.822%), 13 pgs degraded (PG_DEGRADED) 2024-09-10T03:38:26.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:26 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-activate[59056]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-10T03:38:26.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:26 smithi164 bash[59042]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-10T03:38:26.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:26 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-activate[59056]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-10T03:38:26.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:26 smithi164 bash[59042]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-10T03:38:26.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:26 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-activate[59056]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-10T03:38:26.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:26 smithi164 bash[59042]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-10T03:38:26.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:26 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-activate[59056]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-10T03:38:26.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:26 smithi164 bash[59042]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-10T03:38:26.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:26 smithi164 podman[59042]: 2024-09-10 03:38:26.471747645 +0000 UTC m=+1.616492224 container died da26e6434d0c003446a2570f73464f14e29303955146cd4b3b35eaf90d2e6a21 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-10T03:38:27.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 256645 9.80 MB/sec execute 576 sec latency 34.344 ms 2024-09-10T03:38:27.475 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:27 smithi130 ceph-mon[63591]: pgmap v288: 97 pgs: 3 active+undersized, 25 peering, 13 active+undersized+degraded, 56 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s wr, 459 op/s; 140/1587 objects degraded (8.822%) 2024-09-10T03:38:27.519 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:27 smithi164 ceph-mon[49318]: pgmap v288: 97 pgs: 3 active+undersized, 25 peering, 13 active+undersized+degraded, 56 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s wr, 459 op/s; 140/1587 objects degraded (8.822%) 2024-09-10T03:38:27.520 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:27 smithi164 podman[59181]: 2024-09-10 03:38:27.260537145 +0000 UTC m=+0.781273698 container remove da26e6434d0c003446a2570f73464f14e29303955146cd4b3b35eaf90d2e6a21 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-10T03:38:27.520 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:27 smithi164 podman[59240]: 2024-09-10 03:38:27.385896141 +0000 UTC m=+0.018848713 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:38:27.807 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:27 smithi164 podman[59240]: 2024-09-10 03:38:27.519766356 +0000 UTC m=+0.152718912 container create 69f0162ef515ee3e19c9b2c551abab11be7ec9777541799041767e6e5f4a6d66 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, ceph=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-10T03:38:28.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 256980 9.79 MB/sec execute 577 sec latency 47.295 ms 2024-09-10T03:38:28.208 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:27 smithi164 podman[59240]: 2024-09-10 03:38:27.806759012 +0000 UTC m=+0.439711582 container init 69f0162ef515ee3e19c9b2c551abab11be7ec9777541799041767e6e5f4a6d66 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-10T03:38:28.208 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:27 smithi164 podman[59240]: 2024-09-10 03:38:27.810704137 +0000 UTC m=+0.443656701 container start 69f0162ef515ee3e19c9b2c551abab11be7ec9777541799041767e6e5f4a6d66 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-10T03:38:28.208 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:27 smithi164 ceph-osd[59258]: -- 172.21.15.164:0/1808993443 <== mon.1 v2:172.21.15.164:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c2634b01e0 con 0x55c26343c000 2024-09-10T03:38:28.208 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:27 smithi164 bash[59240]: 69f0162ef515ee3e19c9b2c551abab11be7ec9777541799041767e6e5f4a6d66 2024-09-10T03:38:28.208 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:27 smithi164 systemd[1]: Started Ceph osd.7 for 5829498e-6f23-11ef-bcea-c7b262605968. 2024-09-10T03:38:28.996 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:28 smithi130 ceph-mon[63591]: pgmap v289: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.9 MiB/s wr, 564 op/s; 267/1689 objects degraded (15.808%) 2024-09-10T03:38:28.996 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:28 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:28.996 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:28 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:28.996 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:28 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:38:28.996 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:28 smithi130 ceph-mon[63591]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-09-10T03:38:29.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 257328 9.79 MB/sec execute 578 sec latency 72.694 ms 2024-09-10T03:38:29.432 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:28 smithi164 ceph-mon[49318]: pgmap v289: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.9 MiB/s wr, 564 op/s; 267/1689 objects degraded (15.808%) 2024-09-10T03:38:29.433 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:28 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:29.433 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:28 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:29.433 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:28 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:38:29.433 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:28 smithi164 ceph-mon[49318]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-09-10T03:38:29.700 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:29 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:38:29.431+0000 7f73dd42e640 -1 Falling back to public interface 2024-09-10T03:38:30.128 INFO:teuthology.orchestra.run.smithi130.stdout: 5 257662 9.78 MB/sec execute 579 sec latency 71.439 ms 2024-09-10T03:38:31.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 258044 9.78 MB/sec execute 580 sec latency 40.642 ms 2024-09-10T03:38:31.160 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:31 smithi130 ceph-mon[63591]: pgmap v290: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 347 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.8 MiB/s wr, 487 op/s; 265/1701 objects degraded (15.579%) 2024-09-10T03:38:31.160 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:31 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:31.160 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:31 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:38:31.352 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:31 smithi164 ceph-mon[49318]: pgmap v290: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 347 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.8 MiB/s wr, 487 op/s; 265/1701 objects degraded (15.579%) 2024-09-10T03:38:31.352 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:31 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:31.352 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:31 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:38:32.128 INFO:teuthology.orchestra.run.smithi130.stdout: 5 258535 9.79 MB/sec execute 581 sec latency 39.972 ms 2024-09-10T03:38:33.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 259000 9.79 MB/sec execute 582 sec latency 43.072 ms 2024-09-10T03:38:33.212 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:32 smithi164 ceph-mon[49318]: pgmap v291: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 483 op/s; 266/1665 objects degraded (15.976%) 2024-09-10T03:38:33.212 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:32 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:33.212 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:32 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:33.213 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:32 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 266/1665 objects degraded (15.976%), 35 pgs degraded (PG_DEGRADED) 2024-09-10T03:38:33.213 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:32 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:33.213 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:32 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:33.219 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:32 smithi130 ceph-mon[63591]: pgmap v291: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 483 op/s; 266/1665 objects degraded (15.976%) 2024-09-10T03:38:33.219 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:32 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:33.219 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:32 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:33.219 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:32 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 266/1665 objects degraded (15.976%), 35 pgs degraded (PG_DEGRADED) 2024-09-10T03:38:33.219 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:32 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:33.219 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:32 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:34.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 259326 9.79 MB/sec execute 583 sec latency 37.549 ms 2024-09-10T03:38:35.010 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:34 smithi130 ceph-mon[63591]: pgmap v292: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.5 MiB/s wr, 476 op/s; 256/1665 objects degraded (15.375%) 2024-09-10T03:38:35.127 INFO:teuthology.orchestra.run.smithi130.stdout: 5 259690 9.80 MB/sec execute 584 sec latency 36.265 ms 2024-09-10T03:38:35.297 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:34 smithi164 ceph-mon[49318]: pgmap v292: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.5 MiB/s wr, 476 op/s; 256/1665 objects degraded (15.375%) 2024-09-10T03:38:36.128 INFO:teuthology.orchestra.run.smithi130.stdout: 5 260051 9.80 MB/sec execute 585 sec latency 48.375 ms 2024-09-10T03:38:37.032 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:36 smithi130 ceph-mon[63591]: pgmap v293: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 345 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 5.3 MiB/s wr, 502 op/s; 259/1641 objects degraded (15.783%) 2024-09-10T03:38:37.032 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:36 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:37.032 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:36 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:37.033 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:36 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:38:37.033 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:36 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:38:37.033 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:36 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:37.033 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:36 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:37.033 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:36 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:38:37.033 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:36 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:38:37.033 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:36 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:38:37.033 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:38:37.033 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:38:37.033 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:37.033 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-10T03:38:37.033 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-10T03:38:37.033 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-10T03:38:37.034 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-10T03:38:37.034 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-10T03:38:37.034 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-10T03:38:37.034 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-10T03:38:37.034 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-10T03:38:37.034 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-10T03:38:37.034 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-10T03:38:37.034 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-10T03:38:37.034 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-10T03:38:37.034 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-10T03:38:37.034 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-10T03:38:37.034 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-10T03:38:37.034 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-10T03:38:37.034 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-10T03:38:37.128 INFO:teuthology.orchestra.run.smithi130.stdout: 5 260353 9.80 MB/sec execute 586 sec latency 48.744 ms 2024-09-10T03:38:37.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: pgmap v293: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 345 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 5.3 MiB/s wr, 502 op/s; 259/1641 objects degraded (15.783%) 2024-09-10T03:38:37.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:37.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:37.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:38:37.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:38:37.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:37.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:37.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:38:37.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:38:37.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:38:37.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:38:37.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:38:37.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:37.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-10T03:38:37.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-10T03:38:37.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-10T03:38:37.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-10T03:38:37.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-10T03:38:37.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-10T03:38:37.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-10T03:38:37.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-10T03:38:37.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-10T03:38:37.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-10T03:38:37.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-10T03:38:37.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-10T03:38:37.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-10T03:38:37.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-10T03:38:37.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-10T03:38:37.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-10T03:38:37.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-10T03:38:38.052 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:38 smithi130 ceph-mon[63591]: Upgrade: Setting container_image for all osd 2024-09-10T03:38:38.052 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:38 smithi130 ceph-mon[63591]: Upgrade: Setting require_osd_release to 19 squid 2024-09-10T03:38:38.052 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:38 smithi130 ceph-mon[63591]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-10T03:38:38.052 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:38 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-10T03:38:38.052 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:38 smithi130 ceph-mon[63591]: osdmap e151: 8 total, 7 up, 8 in 2024-09-10T03:38:38.052 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:38 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi130.ukcxnb"]} : dispatch 2024-09-10T03:38:38.128 INFO:teuthology.orchestra.run.smithi130.stdout: 5 260746 9.80 MB/sec execute 587 sec latency 61.318 ms 2024-09-10T03:38:38.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:38 smithi164 ceph-mon[49318]: Upgrade: Setting container_image for all osd 2024-09-10T03:38:38.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:38 smithi164 ceph-mon[49318]: Upgrade: Setting require_osd_release to 19 squid 2024-09-10T03:38:38.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:38 smithi164 ceph-mon[49318]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-10T03:38:38.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:38 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-10T03:38:38.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:38 smithi164 ceph-mon[49318]: osdmap e151: 8 total, 7 up, 8 in 2024-09-10T03:38:38.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:38 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi130.ukcxnb"]} : dispatch 2024-09-10T03:38:38.957 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:38 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:38:38.620+0000 7f73dd42e640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-10T03:38:39.060 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:39 smithi130 ceph-mon[63591]: Upgrade: It appears safe to stop mds.foofs.smithi130.ukcxnb 2024-09-10T03:38:39.060 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:39 smithi130 ceph-mon[63591]: pgmap v295: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.9 MiB/s wr, 594 op/s; 267/1704 objects degraded (15.669%) 2024-09-10T03:38:39.060 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:39 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 267/1704 objects degraded (15.669%), 35 pgs degraded (PG_DEGRADED) 2024-09-10T03:38:39.128 INFO:teuthology.orchestra.run.smithi130.stdout: 5 261097 9.79 MB/sec execute 588 sec latency 46.155 ms 2024-09-10T03:38:39.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:39 smithi164 ceph-mon[49318]: Upgrade: It appears safe to stop mds.foofs.smithi130.ukcxnb 2024-09-10T03:38:39.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:39 smithi164 ceph-mon[49318]: pgmap v295: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.9 MiB/s wr, 594 op/s; 267/1704 objects degraded (15.669%) 2024-09-10T03:38:39.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:39 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 267/1704 objects degraded (15.669%), 35 pgs degraded (PG_DEGRADED) 2024-09-10T03:38:40.128 INFO:teuthology.orchestra.run.smithi130.stdout: 5 261418 9.79 MB/sec execute 589 sec latency 44.045 ms 2024-09-10T03:38:40.457 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:40 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:38:40.166+0000 7f73dd42e640 -1 osd.7 148 log_to_monitors true 2024-09-10T03:38:41.116 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:41 smithi130 ceph-mon[63591]: pgmap v296: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.8 MiB/s wr, 594 op/s; 271/1722 objects degraded (15.738%) 2024-09-10T03:38:41.117 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:41 smithi130 ceph-mon[63591]: from='osd.7 [v2:172.21.15.164:6824/3893571563,v1:172.21.15.164:6825/3893571563]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-10T03:38:41.117 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:41 smithi130 ceph-mon[63591]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-10T03:38:41.117 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:41 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:41.117 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:41 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi130.ukcxnb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-10T03:38:41.117 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:41 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:38:41.128 INFO:teuthology.orchestra.run.smithi130.stdout: 5 261704 9.79 MB/sec execute 590 sec latency 61.530 ms 2024-09-10T03:38:41.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:41 smithi164 ceph-mon[49318]: pgmap v296: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.8 MiB/s wr, 594 op/s; 271/1722 objects degraded (15.738%) 2024-09-10T03:38:41.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:41 smithi164 ceph-mon[49318]: from='osd.7 [v2:172.21.15.164:6824/3893571563,v1:172.21.15.164:6825/3893571563]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-10T03:38:41.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:41 smithi164 ceph-mon[49318]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-10T03:38:41.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:41 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:41.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:41 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi130.ukcxnb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-10T03:38:41.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:41 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:38:41.957 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:38:41 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:38:41.464+0000 7f73d11d0640 -1 osd.7 148 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-10T03:38:42.128 INFO:teuthology.orchestra.run.smithi130.stdout: 5 262062 9.79 MB/sec execute 591 sec latency 179.859 ms 2024-09-10T03:38:42.130 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:42 smithi130 ceph-mon[63591]: Upgrade: Updating mds.foofs.smithi130.ukcxnb 2024-09-10T03:38:42.131 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:42 smithi130 ceph-mon[63591]: Deploying daemon mds.foofs.smithi130.ukcxnb on smithi130 2024-09-10T03:38:42.131 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:42 smithi130 ceph-mon[63591]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-10T03:38:42.131 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:42 smithi130 ceph-mon[63591]: osdmap e152: 8 total, 7 up, 8 in 2024-09-10T03:38:42.131 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:42 smithi130 ceph-mon[63591]: from='osd.7 [v2:172.21.15.164:6824/3893571563,v1:172.21.15.164:6825/3893571563]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]} : dispatch 2024-09-10T03:38:42.131 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:42 smithi130 ceph-mon[63591]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]} : dispatch 2024-09-10T03:38:42.131 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:42 smithi130 ceph-mon[63591]: from='osd.7 ' entity='osd.7' 2024-09-10T03:38:42.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:42 smithi164 ceph-mon[49318]: Upgrade: Updating mds.foofs.smithi130.ukcxnb 2024-09-10T03:38:42.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:42 smithi164 ceph-mon[49318]: Deploying daemon mds.foofs.smithi130.ukcxnb on smithi130 2024-09-10T03:38:42.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:42 smithi164 ceph-mon[49318]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-10T03:38:42.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:42 smithi164 ceph-mon[49318]: osdmap e152: 8 total, 7 up, 8 in 2024-09-10T03:38:42.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:42 smithi164 ceph-mon[49318]: from='osd.7 [v2:172.21.15.164:6824/3893571563,v1:172.21.15.164:6825/3893571563]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]} : dispatch 2024-09-10T03:38:42.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:42 smithi164 ceph-mon[49318]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]} : dispatch 2024-09-10T03:38:42.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:42 smithi164 ceph-mon[49318]: from='osd.7 ' entity='osd.7' 2024-09-10T03:38:43.128 INFO:teuthology.orchestra.run.smithi130.stdout: 5 262227 9.78 MB/sec execute 592 sec latency 673.110 ms 2024-09-10T03:38:43.232 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:43 smithi130 ceph-mon[63591]: pgmap v298: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.4 KiB/s rd, 5.7 MiB/s wr, 514 op/s; 264/1695 objects degraded (15.575%) 2024-09-10T03:38:43.232 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:43 smithi130 ceph-mon[63591]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-10T03:38:43.232 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:43 smithi130 ceph-mon[63591]: osd.7 [v2:172.21.15.164:6824/3893571563,v1:172.21.15.164:6825/3893571563] boot 2024-09-10T03:38:43.232 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:43 smithi130 ceph-mon[63591]: osdmap e153: 8 total, 8 up, 8 in 2024-09-10T03:38:43.232 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:43 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-10T03:38:43.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:43 smithi164 ceph-mon[49318]: pgmap v298: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.4 KiB/s rd, 5.7 MiB/s wr, 514 op/s; 264/1695 objects degraded (15.575%) 2024-09-10T03:38:43.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:43 smithi164 ceph-mon[49318]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-10T03:38:43.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:43 smithi164 ceph-mon[49318]: osd.7 [v2:172.21.15.164:6824/3893571563,v1:172.21.15.164:6825/3893571563] boot 2024-09-10T03:38:43.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:43 smithi164 ceph-mon[49318]: osdmap e153: 8 total, 8 up, 8 in 2024-09-10T03:38:43.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:43 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-10T03:38:43.957 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:38:43 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:38:43.675+0000 7fc38d863640 -1 bdev(0x563ce3124c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 4 2024-09-10T03:38:44.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:43 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130[63587]: 2024-09-10T03:38:43.592+0000 7ff02e305640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-10T03:38:44.128 INFO:teuthology.orchestra.run.smithi130.stdout: 5 262227 9.76 MB/sec execute 593 sec latency 1673.383 ms 2024-09-10T03:38:44.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:44 smithi130 ceph-mon[63591]: osdmap e154: 8 total, 8 up, 8 in 2024-09-10T03:38:44.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:44 smithi130 ceph-mon[63591]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-10T03:38:44.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:44 smithi130 ceph-mon[63591]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-10T03:38:44.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:44 smithi130 ceph-mon[63591]: osdmap e155: 8 total, 8 up, 8 in 2024-09-10T03:38:44.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:44 smithi130 ceph-mon[63591]: Standby daemon mds.foofs.smithi164.awiezo assigned to filesystem foofs as rank 0 2024-09-10T03:38:44.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:44 smithi130 ceph-mon[63591]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-10T03:38:44.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:44 smithi130 ceph-mon[63591]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-10T03:38:44.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:44 smithi130 ceph-mon[63591]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-10T03:38:44.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:44 smithi130 ceph-mon[63591]: fsmap foofs:1/1 {0=foofs.smithi164.awiezo=up:replay} 2024-09-10T03:38:44.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:44 smithi130 ceph-mon[63591]: pgmap v302: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 4.5 MiB/s wr, 552 op/s; 260/1629 objects degraded (15.961%) 2024-09-10T03:38:44.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:44 smithi164 ceph-mon[49318]: osdmap e154: 8 total, 8 up, 8 in 2024-09-10T03:38:44.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:44 smithi164 ceph-mon[49318]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-10T03:38:44.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:44 smithi164 ceph-mon[49318]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-10T03:38:44.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:44 smithi164 ceph-mon[49318]: osdmap e155: 8 total, 8 up, 8 in 2024-09-10T03:38:44.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:44 smithi164 ceph-mon[49318]: Standby daemon mds.foofs.smithi164.awiezo assigned to filesystem foofs as rank 0 2024-09-10T03:38:44.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:44 smithi164 ceph-mon[49318]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-10T03:38:44.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:44 smithi164 ceph-mon[49318]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-10T03:38:44.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:44 smithi164 ceph-mon[49318]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-10T03:38:44.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:44 smithi164 ceph-mon[49318]: fsmap foofs:1/1 {0=foofs.smithi164.awiezo=up:replay} 2024-09-10T03:38:44.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:44 smithi164 ceph-mon[49318]: pgmap v302: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 4.5 MiB/s wr, 552 op/s; 260/1629 objects degraded (15.961%) 2024-09-10T03:38:45.128 INFO:teuthology.orchestra.run.smithi130.stdout: 5 262227 9.74 MB/sec execute 594 sec latency 2673.433 ms 2024-09-10T03:38:45.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:45 smithi130 ceph-mon[63591]: Health check update: Degraded data redundancy: 260/1629 objects degraded (15.961%), 35 pgs degraded (PG_DEGRADED) 2024-09-10T03:38:45.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:45 smithi130 ceph-mon[63591]: osdmap e156: 8 total, 8 up, 8 in 2024-09-10T03:38:45.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:45 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:45.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:45 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:38:45.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:45 smithi164 ceph-mon[49318]: Health check update: Degraded data redundancy: 260/1629 objects degraded (15.961%), 35 pgs degraded (PG_DEGRADED) 2024-09-10T03:38:45.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:45 smithi164 ceph-mon[49318]: osdmap e156: 8 total, 8 up, 8 in 2024-09-10T03:38:45.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:45 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:45.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:45 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:38:46.128 INFO:teuthology.orchestra.run.smithi130.stdout: 5 262227 9.73 MB/sec execute 595 sec latency 3673.484 ms 2024-09-10T03:38:46.752 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:46 smithi164 ceph-mon[49318]: reconnect by client.24343 172.21.15.164:0/3206963038 after 0 2024-09-10T03:38:46.752 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:46 smithi164 ceph-mon[49318]: reconnect by client.14556 172.21.15.130:0/2159933080 after 0.000999983 2024-09-10T03:38:46.752 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:46 smithi164 ceph-mon[49318]: mds.? [v2:172.21.15.164:6832/3151002352,v1:172.21.15.164:6833/3151002352] up:reconnect 2024-09-10T03:38:46.752 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:46 smithi164 ceph-mon[49318]: fsmap foofs:1/1 {0=foofs.smithi164.awiezo=up:reconnect} 2024-09-10T03:38:46.752 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:46 smithi164 ceph-mon[49318]: osdmap e157: 8 total, 8 up, 8 in 2024-09-10T03:38:46.752 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:46 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds metadata", "who": "foofs.smithi130.ukcxnb"} : dispatch 2024-09-10T03:38:46.752 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:46 smithi164 ceph-mon[49318]: pgmap v305: 97 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 4 active+undersized, 29 active+undersized+degraded, 59 active+clean; 341 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 437 KiB/s wr, 41 op/s; 210/1599 objects degraded (13.133%); 446 KiB/s, 29 objects/s recovering 2024-09-10T03:38:46.752 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:46 smithi164 ceph-mon[49318]: mds.? [v2:172.21.15.164:6832/3151002352,v1:172.21.15.164:6833/3151002352] up:rejoin 2024-09-10T03:38:46.752 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:46 smithi164 ceph-mon[49318]: fsmap foofs:1/1 {0=foofs.smithi164.awiezo=up:rejoin} 2024-09-10T03:38:46.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:46 smithi130 ceph-mon[63591]: reconnect by client.24343 172.21.15.164:0/3206963038 after 0 2024-09-10T03:38:46.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:46 smithi130 ceph-mon[63591]: reconnect by client.14556 172.21.15.130:0/2159933080 after 0.000999983 2024-09-10T03:38:46.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:46 smithi130 ceph-mon[63591]: mds.? [v2:172.21.15.164:6832/3151002352,v1:172.21.15.164:6833/3151002352] up:reconnect 2024-09-10T03:38:46.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:46 smithi130 ceph-mon[63591]: fsmap foofs:1/1 {0=foofs.smithi164.awiezo=up:reconnect} 2024-09-10T03:38:46.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:46 smithi130 ceph-mon[63591]: osdmap e157: 8 total, 8 up, 8 in 2024-09-10T03:38:46.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:46 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds metadata", "who": "foofs.smithi130.ukcxnb"} : dispatch 2024-09-10T03:38:46.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:46 smithi130 ceph-mon[63591]: pgmap v305: 97 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 4 active+undersized, 29 active+undersized+degraded, 59 active+clean; 341 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 437 KiB/s wr, 41 op/s; 210/1599 objects degraded (13.133%); 446 KiB/s, 29 objects/s recovering 2024-09-10T03:38:46.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:46 smithi130 ceph-mon[63591]: mds.? [v2:172.21.15.164:6832/3151002352,v1:172.21.15.164:6833/3151002352] up:rejoin 2024-09-10T03:38:46.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:46 smithi130 ceph-mon[63591]: fsmap foofs:1/1 {0=foofs.smithi164.awiezo=up:rejoin} 2024-09-10T03:38:47.128 INFO:teuthology.orchestra.run.smithi130.stdout: 5 262227 9.71 MB/sec execute 596 sec latency 4673.534 ms 2024-09-10T03:38:47.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:47 smithi164 ceph-mon[49318]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-10T03:38:47.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:47 smithi164 ceph-mon[49318]: mds.? [v2:172.21.15.164:6832/3151002352,v1:172.21.15.164:6833/3151002352] up:clientreplay 2024-09-10T03:38:47.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:47 smithi164 ceph-mon[49318]: mds.? [v2:172.21.15.130:6834/493682062,v1:172.21.15.130:6835/493682062] up:boot 2024-09-10T03:38:47.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:47 smithi164 ceph-mon[49318]: fsmap foofs:1/1 {0=foofs.smithi164.awiezo=up:clientreplay} 1 up:standby 2024-09-10T03:38:47.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:47 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds metadata", "who": "foofs.smithi130.ukcxnb"} : dispatch 2024-09-10T03:38:47.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:47 smithi164 ceph-mon[49318]: daemon mds.foofs.smithi164.awiezo is now active in filesystem foofs as rank 0 2024-09-10T03:38:48.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:47 smithi130 ceph-mon[63591]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-10T03:38:48.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:47 smithi130 ceph-mon[63591]: mds.? [v2:172.21.15.164:6832/3151002352,v1:172.21.15.164:6833/3151002352] up:clientreplay 2024-09-10T03:38:48.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:47 smithi130 ceph-mon[63591]: mds.? [v2:172.21.15.130:6834/493682062,v1:172.21.15.130:6835/493682062] up:boot 2024-09-10T03:38:48.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:47 smithi130 ceph-mon[63591]: fsmap foofs:1/1 {0=foofs.smithi164.awiezo=up:clientreplay} 1 up:standby 2024-09-10T03:38:48.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:47 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds metadata", "who": "foofs.smithi130.ukcxnb"} : dispatch 2024-09-10T03:38:48.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:47 smithi130 ceph-mon[63591]: daemon mds.foofs.smithi164.awiezo is now active in filesystem foofs as rank 0 2024-09-10T03:38:48.128 INFO:teuthology.orchestra.run.smithi130.stdout: 5 262228 9.69 MB/sec execute 597 sec latency 5673.600 ms 2024-09-10T03:38:48.749 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:48 smithi130 ceph-mon[63591]: pgmap v306: 97 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 93 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 50 MiB/s rd, 1.2 MiB/s wr, 154 op/s; 28/1602 objects degraded (1.748%); 4.3 MiB/s, 75 objects/s recovering 2024-09-10T03:38:48.749 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:48 smithi130 ceph-mon[63591]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-10T03:38:48.750 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:48 smithi130 ceph-mon[63591]: mds.? [v2:172.21.15.164:6832/3151002352,v1:172.21.15.164:6833/3151002352] up:active 2024-09-10T03:38:48.750 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:48 smithi130 ceph-mon[63591]: fsmap foofs:1 {0=foofs.smithi164.awiezo=up:active} 1 up:standby 2024-09-10T03:38:48.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:48 smithi164 ceph-mon[49318]: pgmap v306: 97 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 93 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 50 MiB/s rd, 1.2 MiB/s wr, 154 op/s; 28/1602 objects degraded (1.748%); 4.3 MiB/s, 75 objects/s recovering 2024-09-10T03:38:48.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:48 smithi164 ceph-mon[49318]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-10T03:38:48.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:48 smithi164 ceph-mon[49318]: mds.? [v2:172.21.15.164:6832/3151002352,v1:172.21.15.164:6833/3151002352] up:active 2024-09-10T03:38:48.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:48 smithi164 ceph-mon[49318]: fsmap foofs:1 {0=foofs.smithi164.awiezo=up:active} 1 up:standby 2024-09-10T03:38:49.128 INFO:teuthology.orchestra.run.smithi130.stdout: 5 262465 9.69 MB/sec execute 598 sec latency 6030.066 ms 2024-09-10T03:38:50.029 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:49 smithi130 ceph-mon[63591]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/1602 objects degraded (1.748%), 1 pg degraded) 2024-09-10T03:38:50.029 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:49 smithi130 ceph-mon[63591]: Cluster is now healthy 2024-09-10T03:38:50.128 INFO:teuthology.orchestra.run.smithi130.stdout: 5 262677 9.68 MB/sec execute 599 sec latency 62.997 ms 2024-09-10T03:38:50.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:49 smithi164 ceph-mon[49318]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/1602 objects degraded (1.748%), 1 pg degraded) 2024-09-10T03:38:50.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:49 smithi164 ceph-mon[49318]: Cluster is now healthy 2024-09-10T03:38:50.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:50 smithi164 ceph-mon[49318]: pgmap v307: 97 pgs: 2 peering, 95 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 977 KiB/s wr, 112 op/s; 3.0 MiB/s, 57 objects/s recovering 2024-09-10T03:38:51.094 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:50 smithi130 ceph-mon[63591]: pgmap v307: 97 pgs: 2 peering, 95 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 977 KiB/s wr, 112 op/s; 3.0 MiB/s, 57 objects/s recovering 2024-09-10T03:38:51.128 INFO:teuthology.orchestra.run.smithi130.stdout: 5 cleanup 600 sec 2024-09-10T03:38:51.334 INFO:teuthology.orchestra.run.smithi130.stdout: 0 cleanup 600 sec 2024-09-10T03:38:51.334 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:38:51.334 INFO:teuthology.orchestra.run.smithi130.stdout: Operation Count AvgLat MaxLat 2024-09-10T03:38:51.334 INFO:teuthology.orchestra.run.smithi130.stdout: ---------------------------------------- 2024-09-10T03:38:51.334 INFO:teuthology.orchestra.run.smithi130.stdout: NTCreateX 186199 4.539 2150.168 2024-09-10T03:38:51.334 INFO:teuthology.orchestra.run.smithi130.stdout: Close 136858 0.135 10.788 2024-09-10T03:38:51.335 INFO:teuthology.orchestra.run.smithi130.stdout: Rename 7887 9.215 2140.310 2024-09-10T03:38:51.335 INFO:teuthology.orchestra.run.smithi130.stdout: Unlink 37524 7.105 2151.416 2024-09-10T03:38:51.335 INFO:teuthology.orchestra.run.smithi130.stdout: Qpathinfo 168996 1.218 6003.494 2024-09-10T03:38:51.335 INFO:teuthology.orchestra.run.smithi130.stdout: Qfileinfo 29500 0.007 1.647 2024-09-10T03:38:51.335 INFO:teuthology.orchestra.run.smithi130.stdout: Qfsinfo 30895 0.907 1963.103 2024-09-10T03:38:51.335 INFO:teuthology.orchestra.run.smithi130.stdout: Sfileinfo 15161 12.804 1092.852 2024-09-10T03:38:51.335 INFO:teuthology.orchestra.run.smithi130.stdout: Find 65257 0.956 21.362 2024-09-10T03:38:51.335 INFO:teuthology.orchestra.run.smithi130.stdout: WriteX 92062 14.121 6030.061 2024-09-10T03:38:51.335 INFO:teuthology.orchestra.run.smithi130.stdout: ReadX 292279 0.010 10.720 2024-09-10T03:38:51.335 INFO:teuthology.orchestra.run.smithi130.stdout: LockX 608 0.412 20.666 2024-09-10T03:38:51.335 INFO:teuthology.orchestra.run.smithi130.stdout: UnlockX 608 0.352 15.661 2024-09-10T03:38:51.335 INFO:teuthology.orchestra.run.smithi130.stdout: Flush 13016 0.002 0.090 2024-09-10T03:38:51.335 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:38:51.335 INFO:teuthology.orchestra.run.smithi130.stdout:Throughput 9.68083 MB/sec 5 clients 5 procs max_latency=6030.066 ms 2024-09-10T03:38:51.337 DEBUG:teuthology.orchestra.run.smithi130:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-10T03:38:51.372 DEBUG:teuthology.orchestra.run.smithi130:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-10T03:38:51.498 DEBUG:teuthology.orchestra.run.smithi130:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-10T03:38:51.523 INFO:teuthology.orchestra.run.smithi130.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-10T03:38:51.523 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:38:51.523 INFO:teuthology.orchestra.run.smithi130.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-10T03:38:51.524 INFO:teuthology.orchestra.run.smithi130.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-10T03:38:51.624 INFO:teuthology.orchestra.run.smithi130.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-10T03:38:51.624 INFO:teuthology.orchestra.run.smithi130.stdout:releasing clients 2024-09-10T03:38:52.922 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:52 smithi130 ceph-mon[63591]: pgmap v308: 97 pgs: 97 active+clean; 338 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s rd, 1.8 MiB/s wr, 219 op/s; 2.4 MiB/s, 44 objects/s recovering 2024-09-10T03:38:53.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:52 smithi164 ceph-mon[49318]: pgmap v308: 97 pgs: 97 active+clean; 338 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s rd, 1.8 MiB/s wr, 219 op/s; 2.4 MiB/s, 44 objects/s recovering 2024-09-10T03:38:53.624 INFO:teuthology.orchestra.run.smithi130.stdout: 5 397 22.89 MB/sec execute 1 sec latency 96.000 ms 2024-09-10T03:38:54.624 INFO:teuthology.orchestra.run.smithi130.stdout: 5 472 17.56 MB/sec execute 2 sec latency 127.074 ms 2024-09-10T03:38:55.063 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:54 smithi130 ceph-mon[63591]: pgmap v309: 97 pgs: 97 active+clean; 345 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 2.4 MiB/s wr, 197 op/s; 2.0 MiB/s, 33 objects/s recovering 2024-09-10T03:38:55.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:54 smithi164 ceph-mon[49318]: pgmap v309: 97 pgs: 97 active+clean; 345 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 2.4 MiB/s wr, 197 op/s; 2.0 MiB/s, 33 objects/s recovering 2024-09-10T03:38:55.393 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:38:55.624 INFO:teuthology.orchestra.run.smithi130.stdout: 5 629 18.60 MB/sec execute 3 sec latency 98.155 ms 2024-09-10T03:38:56.372 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:56 smithi130 ceph-mon[63591]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:38:56.461 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:38:56.461 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (9m) 2m ago 18m 19.4M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:38:56.461 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (6m) 2m ago 18m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 8c7cfb7e87a1 2024-09-10T03:38:56.461 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (6m) 24s ago 18m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 243c094648b8 2024-09-10T03:38:56.461 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (9m) 2m ago 18m 55.9M - dad864ee21e9 72ba01ac1871 2024-09-10T03:38:56.461 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (12m) 2m ago 12m 37.4M - 17.2.0 e1d6a67b021e d98551f8ae50 2024-09-10T03:38:56.461 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (12m) 24s ago 12m 17.0M - 17.2.0 e1d6a67b021e fefa329410be 2024-09-10T03:38:56.462 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (8m) 2m ago 19m 519M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:38:56.462 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283,8765 running (7m) 24s ago 18m 457M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e4034e8e2a6d 2024-09-10T03:38:56.462 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (7m) 2m ago 20m 53.2M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 22e5cf56527a 2024-09-10T03:38:56.462 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (7m) 24s ago 17m 46.2M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 7d80bd0d15b9 2024-09-10T03:38:56.462 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (12m) 2m ago 12m 284M - 4.0 e1d6a67b021e 98e1d2b5cce7 2024-09-10T03:38:56.462 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (12m) 24s ago 12m 88.0M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:38:56.462 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (9m) 2m ago 18m 10.7M - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:38:56.462 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (8m) 24s ago 17m 10.9M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:38:56.462 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (5m) 2m ago 17m 353M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b de3cbff0098b 2024-09-10T03:38:56.462 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (4m) 2m ago 16m 273M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f0722a6d1c90 2024-09-10T03:38:56.462 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (3m) 2m ago 16m 209M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b a871d2ccf96e 2024-09-10T03:38:56.462 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (3m) 2m ago 15m 14.2M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f2bc83e91fa2 2024-09-10T03:38:56.462 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (2m) 24s ago 15m 334M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 5e1e1b5e1266 2024-09-10T03:38:56.462 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (102s) 24s ago 14m 301M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 76ab1aa88b44 2024-09-10T03:38:56.463 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (66s) 24s ago 14m 253M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b ca90894c3878 2024-09-10T03:38:56.463 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (29s) 24s ago 14m 11.9M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 69f0162ef515 2024-09-10T03:38:56.463 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (8m) 2m ago 17m 54.5M - 2.51.0 1d3b7f56885b f59b02fac5b3 2024-09-10T03:38:56.624 INFO:teuthology.orchestra.run.smithi130.stdout: 5 888 15.21 MB/sec execute 4 sec latency 36.317 ms 2024-09-10T03:38:56.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:56 smithi164 ceph-mon[49318]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:38:57.141 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:38:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:38:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:38:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:38:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:38:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:38:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:38:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:38:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 8 2024-09-10T03:38:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:38:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:38:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-10T03:38:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 1 2024-09-10T03:38:57.143 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:38:57.143 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:38:57.143 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-10T03:38:57.143 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 13 2024-09-10T03:38:57.143 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:38:57.143 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:38:57.392 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:57 smithi130 ceph-mon[63591]: pgmap v310: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 4.6 MiB/s wr, 207 op/s; 1.7 MiB/s, 25 objects/s recovering 2024-09-10T03:38:57.392 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:57 smithi130 ceph-mon[63591]: from='client.44383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:38:57.392 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:57 smithi130 ceph-mon[63591]: daemon mds.foofs.smithi130.ukcxnb restarted 2024-09-10T03:38:57.392 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:57 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:57.392 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:57 smithi130 ceph-mon[63591]: osdmap e158: 8 total, 8 up, 8 in 2024-09-10T03:38:57.392 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:57 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:57.392 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:57 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:38:57.392 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:57 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/3169603870' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:38:57.623 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:38:57.623 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:38:57.623 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:38:57.624 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:38:57.624 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-09-10T03:38:57.624 INFO:teuthology.orchestra.run.smithi130.stdout: "mon", 2024-09-10T03:38:57.624 INFO:teuthology.orchestra.run.smithi130.stdout: "crash", 2024-09-10T03:38:57.624 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr", 2024-09-10T03:38:57.624 INFO:teuthology.orchestra.run.smithi130.stdout: "osd" 2024-09-10T03:38:57.624 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-09-10T03:38:57.624 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "14/23 daemons upgraded", 2024-09-10T03:38:57.624 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading mds daemons", 2024-09-10T03:38:57.624 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:38:57.624 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:38:57.625 INFO:teuthology.orchestra.run.smithi130.stdout: 5 cleanup 5 sec 2024-09-10T03:38:57.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:57 smithi164 ceph-mon[49318]: pgmap v310: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 4.6 MiB/s wr, 207 op/s; 1.7 MiB/s, 25 objects/s recovering 2024-09-10T03:38:57.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:57 smithi164 ceph-mon[49318]: from='client.44383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:38:57.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:57 smithi164 ceph-mon[49318]: daemon mds.foofs.smithi130.ukcxnb restarted 2024-09-10T03:38:57.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:57 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:57.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:57 smithi164 ceph-mon[49318]: osdmap e158: 8 total, 8 up, 8 in 2024-09-10T03:38:57.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:57 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:38:57.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:57 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:38:57.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:57 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/3169603870' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:38:58.086 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-09-10T03:38:58.454 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:58 smithi130 ceph-mon[63591]: from='client.44387 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:38:58.455 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:58 smithi130 ceph-mon[63591]: mds.? [v2:172.21.15.130:6834/340154171,v1:172.21.15.130:6835/340154171] up:boot 2024-09-10T03:38:58.455 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:58 smithi130 ceph-mon[63591]: fsmap foofs:1 {0=foofs.smithi164.awiezo=up:active} 1 up:standby 2024-09-10T03:38:58.455 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:58 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds metadata", "who": "foofs.smithi130.ukcxnb"} : dispatch 2024-09-10T03:38:58.455 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:58 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/1805149728' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:38:58.624 INFO:teuthology.orchestra.run.smithi130.stdout: 5 cleanup 6 sec 2024-09-10T03:38:58.683 INFO:teuthology.orchestra.run.smithi130.stdout: 0 cleanup 6 sec 2024-09-10T03:38:58.683 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:38:58.683 INFO:teuthology.orchestra.run.smithi130.stdout: Operation Count AvgLat MaxLat 2024-09-10T03:38:58.683 INFO:teuthology.orchestra.run.smithi130.stdout: ---------------------------------------- 2024-09-10T03:38:58.683 INFO:teuthology.orchestra.run.smithi130.stdout: NTCreateX 595 15.508 127.068 2024-09-10T03:38:58.683 INFO:teuthology.orchestra.run.smithi130.stdout: Close 551 8.915 99.214 2024-09-10T03:38:58.683 INFO:teuthology.orchestra.run.smithi130.stdout: Rename 22 28.259 43.964 2024-09-10T03:38:58.683 INFO:teuthology.orchestra.run.smithi130.stdout: Unlink 57 4.200 21.182 2024-09-10T03:38:58.683 INFO:teuthology.orchestra.run.smithi130.stdout: Qpathinfo 230 3.101 66.670 2024-09-10T03:38:58.683 INFO:teuthology.orchestra.run.smithi130.stdout: Qfileinfo 294 18.937 62.710 2024-09-10T03:38:58.683 INFO:teuthology.orchestra.run.smithi130.stdout: Qfsinfo 121 0.942 8.591 2024-09-10T03:38:58.684 INFO:teuthology.orchestra.run.smithi130.stdout: Sfileinfo 99 17.694 39.115 2024-09-10T03:38:58.684 INFO:teuthology.orchestra.run.smithi130.stdout: Find 151 1.169 4.852 2024-09-10T03:38:58.684 INFO:teuthology.orchestra.run.smithi130.stdout: WriteX 1453 0.026 0.088 2024-09-10T03:38:58.684 INFO:teuthology.orchestra.run.smithi130.stdout: ReadX 529 0.025 0.581 2024-09-10T03:38:58.684 INFO:teuthology.orchestra.run.smithi130.stdout: Flush 120 14.535 28.754 2024-09-10T03:38:58.684 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:38:58.684 INFO:teuthology.orchestra.run.smithi130.stdout:Throughput 15.2055 MB/sec 5 clients 5 procs max_latency=127.074 ms 2024-09-10T03:38:58.685 DEBUG:teuthology.parallel:result is None 2024-09-10T03:38:58.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:58 smithi164 ceph-mon[49318]: from='client.44387 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:38:58.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:58 smithi164 ceph-mon[49318]: mds.? [v2:172.21.15.130:6834/340154171,v1:172.21.15.130:6835/340154171] up:boot 2024-09-10T03:38:58.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:58 smithi164 ceph-mon[49318]: fsmap foofs:1 {0=foofs.smithi164.awiezo=up:active} 1 up:standby 2024-09-10T03:38:58.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:58 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds metadata", "who": "foofs.smithi130.ukcxnb"} : dispatch 2024-09-10T03:38:58.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:58 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/1805149728' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:38:59.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:59 smithi164 ceph-mon[49318]: from='client.44395 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:38:59.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:38:59 smithi164 ceph-mon[49318]: pgmap v312: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 452 KiB/s rd, 11 MiB/s wr, 374 op/s; 16 KiB/s, 2 objects/s recovering 2024-09-10T03:38:59.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:59 smithi130 ceph-mon[63591]: from='client.44395 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:38:59.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:38:59 smithi130 ceph-mon[63591]: pgmap v312: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 452 KiB/s rd, 11 MiB/s wr, 374 op/s; 16 KiB/s, 2 objects/s recovering 2024-09-10T03:39:01.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:01 smithi164 ceph-mon[49318]: pgmap v313: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 42 KiB/s rd, 12 MiB/s wr, 378 op/s 2024-09-10T03:39:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:39:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:01 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:01.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:01 smithi130 ceph-mon[63591]: pgmap v313: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 42 KiB/s rd, 12 MiB/s wr, 378 op/s 2024-09-10T03:39:01.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:01.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:39:01.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:01 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:03.242 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:03 smithi130 ceph-mon[63591]: pgmap v314: 97 pgs: 97 active+clean; 282 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 12 MiB/s wr, 330 op/s 2024-09-10T03:39:03.242 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:03 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:03.242 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:03 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:03.242 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:03 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:03.242 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:03 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:03.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:03 smithi164 ceph-mon[49318]: pgmap v314: 97 pgs: 97 active+clean; 282 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 12 MiB/s wr, 330 op/s 2024-09-10T03:39:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:03 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:03 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:03 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:03 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:05.274 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:05 smithi130 ceph-mon[63591]: pgmap v315: 97 pgs: 97 active+clean; 264 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 334 op/s 2024-09-10T03:39:05.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:05 smithi164 ceph-mon[49318]: pgmap v315: 97 pgs: 97 active+clean; 264 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 334 op/s 2024-09-10T03:39:07.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:07 smithi164 ceph-mon[49318]: pgmap v316: 97 pgs: 97 active+clean; 241 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 306 op/s 2024-09-10T03:39:07.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:07 smithi164 ceph-mon[49318]: Detected new or changed devices on smithi130 2024-09-10T03:39:07.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:07 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:07.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:07 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:07.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:07 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:39:07.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:07 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:39:07.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:07 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:07.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:07 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:07.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:07 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:39:07.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:07 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:07.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:07 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:07.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:07 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:07.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:07 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:07.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:07 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi164.awiezo"]} : dispatch 2024-09-10T03:39:07.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:07 smithi130 ceph-mon[63591]: pgmap v316: 97 pgs: 97 active+clean; 241 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 306 op/s 2024-09-10T03:39:07.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:07 smithi130 ceph-mon[63591]: Detected new or changed devices on smithi130 2024-09-10T03:39:07.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:07 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:07.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:07 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:07.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:07 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:39:07.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:07 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:39:07.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:07 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:07.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:07 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:07.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:07 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:39:07.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:07 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:07.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:07 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:07.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:07 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:07.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:07 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:07.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:07 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi164.awiezo"]} : dispatch 2024-09-10T03:39:08.410 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:08 smithi164 ceph-mon[49318]: Upgrade: It appears safe to stop mds.foofs.smithi164.awiezo 2024-09-10T03:39:08.410 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:08 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:08.411 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:08 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi164.awiezo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-10T03:39:08.411 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:08 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:39:08.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:08 smithi130 ceph-mon[63591]: Upgrade: It appears safe to stop mds.foofs.smithi164.awiezo 2024-09-10T03:39:08.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:08 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:08.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:08 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi164.awiezo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-10T03:39:08.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:08 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:39:09.436 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:09 smithi164 ceph-mon[49318]: Upgrade: Updating mds.foofs.smithi164.awiezo 2024-09-10T03:39:09.436 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:09 smithi164 ceph-mon[49318]: Deploying daemon mds.foofs.smithi164.awiezo on smithi164 2024-09-10T03:39:09.436 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:09 smithi164 ceph-mon[49318]: pgmap v317: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.0 MiB/s wr, 107 op/s 2024-09-10T03:39:09.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:09 smithi130 ceph-mon[63591]: Upgrade: Updating mds.foofs.smithi164.awiezo 2024-09-10T03:39:09.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:09 smithi130 ceph-mon[63591]: Deploying daemon mds.foofs.smithi164.awiezo on smithi164 2024-09-10T03:39:09.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:09 smithi130 ceph-mon[63591]: pgmap v317: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.0 MiB/s wr, 107 op/s 2024-09-10T03:39:09.957 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:39:09 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:39:09.722+0000 7fc38f066640 -1 bdev(0x563ce3124c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 2 2024-09-10T03:39:09.958 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:39:09 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:39:09.723+0000 7fc38985b640 -1 bdev(0x563ce3124c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 3 2024-09-10T03:39:09.958 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:39:09 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:39:09.725+0000 7fc388058640 -1 bdev(0x563ce3124c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 4 2024-09-10T03:39:10.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:09 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130[63587]: 2024-09-10T03:39:09.701+0000 7ff02e305640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-10T03:39:10.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:10 smithi164 ceph-mon[49318]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-10T03:39:10.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:10 smithi164 ceph-mon[49318]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-10T03:39:10.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:10 smithi164 ceph-mon[49318]: osdmap e159: 8 total, 8 up, 8 in 2024-09-10T03:39:10.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:10 smithi164 ceph-mon[49318]: Standby daemon mds.foofs.smithi130.ukcxnb assigned to filesystem foofs as rank 0 2024-09-10T03:39:10.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:10 smithi164 ceph-mon[49318]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-10T03:39:10.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:10 smithi164 ceph-mon[49318]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-10T03:39:10.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:10 smithi164 ceph-mon[49318]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-10T03:39:10.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:10 smithi164 ceph-mon[49318]: fsmap foofs:1/1 {0=foofs.smithi130.ukcxnb=up:replay} 2024-09-10T03:39:10.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:10 smithi130 ceph-mon[63591]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-10T03:39:10.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:10 smithi130 ceph-mon[63591]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-10T03:39:10.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:10 smithi130 ceph-mon[63591]: osdmap e159: 8 total, 8 up, 8 in 2024-09-10T03:39:10.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:10 smithi130 ceph-mon[63591]: Standby daemon mds.foofs.smithi130.ukcxnb assigned to filesystem foofs as rank 0 2024-09-10T03:39:10.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:10 smithi130 ceph-mon[63591]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-10T03:39:10.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:10 smithi130 ceph-mon[63591]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-10T03:39:10.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:10 smithi130 ceph-mon[63591]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-10T03:39:10.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:10 smithi130 ceph-mon[63591]: fsmap foofs:1/1 {0=foofs.smithi130.ukcxnb=up:replay} 2024-09-10T03:39:11.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:11 smithi164 ceph-mon[49318]: pgmap v319: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.4 KiB/s rd, 1.3 MiB/s wr, 85 op/s 2024-09-10T03:39:11.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:11 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds metadata", "who": "foofs.smithi164.awiezo"} : dispatch 2024-09-10T03:39:11.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:11 smithi130 ceph-mon[63591]: pgmap v319: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.4 KiB/s rd, 1.3 MiB/s wr, 85 op/s 2024-09-10T03:39:11.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:11 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds metadata", "who": "foofs.smithi164.awiezo"} : dispatch 2024-09-10T03:39:12.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:12 smithi164 ceph-mon[49318]: reconnect by client.14556 172.21.15.130:0/2159933080 after 0 2024-09-10T03:39:12.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:12 smithi164 ceph-mon[49318]: reconnect by client.24343 172.21.15.164:0/3206963038 after 0 2024-09-10T03:39:12.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:12 smithi164 ceph-mon[49318]: mds.? [v2:172.21.15.130:6834/340154171,v1:172.21.15.130:6835/340154171] up:reconnect 2024-09-10T03:39:12.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:12 smithi164 ceph-mon[49318]: fsmap foofs:1/1 {0=foofs.smithi130.ukcxnb=up:reconnect} 2024-09-10T03:39:12.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:12 smithi130 ceph-mon[63591]: reconnect by client.14556 172.21.15.130:0/2159933080 after 0 2024-09-10T03:39:12.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:12 smithi130 ceph-mon[63591]: reconnect by client.24343 172.21.15.164:0/3206963038 after 0 2024-09-10T03:39:12.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:12 smithi130 ceph-mon[63591]: mds.? [v2:172.21.15.130:6834/340154171,v1:172.21.15.130:6835/340154171] up:reconnect 2024-09-10T03:39:12.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:12 smithi130 ceph-mon[63591]: fsmap foofs:1/1 {0=foofs.smithi130.ukcxnb=up:reconnect} 2024-09-10T03:39:13.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:13 smithi164 ceph-mon[49318]: pgmap v320: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 248 KiB/s wr, 32 op/s 2024-09-10T03:39:13.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:13 smithi164 ceph-mon[49318]: mds.? [v2:172.21.15.130:6834/340154171,v1:172.21.15.130:6835/340154171] up:rejoin 2024-09-10T03:39:13.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:13 smithi164 ceph-mon[49318]: fsmap foofs:1/1 {0=foofs.smithi130.ukcxnb=up:rejoin} 2024-09-10T03:39:13.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:13 smithi164 ceph-mon[49318]: daemon mds.foofs.smithi130.ukcxnb is now active in filesystem foofs as rank 0 2024-09-10T03:39:13.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:13 smithi130 ceph-mon[63591]: pgmap v320: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 248 KiB/s wr, 32 op/s 2024-09-10T03:39:13.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:13 smithi130 ceph-mon[63591]: mds.? [v2:172.21.15.130:6834/340154171,v1:172.21.15.130:6835/340154171] up:rejoin 2024-09-10T03:39:13.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:13 smithi130 ceph-mon[63591]: fsmap foofs:1/1 {0=foofs.smithi130.ukcxnb=up:rejoin} 2024-09-10T03:39:13.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:13 smithi130 ceph-mon[63591]: daemon mds.foofs.smithi130.ukcxnb is now active in filesystem foofs as rank 0 2024-09-10T03:39:14.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:14 smithi164 ceph-mon[49318]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-10T03:39:14.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:14 smithi164 ceph-mon[49318]: mds.? [v2:172.21.15.130:6834/340154171,v1:172.21.15.130:6835/340154171] up:active 2024-09-10T03:39:14.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:14 smithi164 ceph-mon[49318]: fsmap foofs:1 {0=foofs.smithi130.ukcxnb=up:active} 2024-09-10T03:39:14.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:14 smithi130 ceph-mon[63591]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-10T03:39:14.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:14 smithi130 ceph-mon[63591]: mds.? [v2:172.21.15.130:6834/340154171,v1:172.21.15.130:6835/340154171] up:active 2024-09-10T03:39:14.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:14 smithi130 ceph-mon[63591]: fsmap foofs:1 {0=foofs.smithi130.ukcxnb=up:active} 2024-09-10T03:39:15.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:15 smithi130 ceph-mon[63591]: pgmap v321: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 12 KiB/s wr, 16 op/s 2024-09-10T03:39:15.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:15 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:15.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:15 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:39:15.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:15 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:15.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:15 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:15.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:15 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:39:15.653 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:15 smithi164 ceph-mon[49318]: pgmap v321: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 12 KiB/s wr, 16 op/s 2024-09-10T03:39:15.653 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:15 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:15.653 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:15 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:39:15.653 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:15 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:15.653 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:15 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:15.653 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:15 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:39:16.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:16 smithi164 ceph-mon[49318]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-10T03:39:16.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:16 smithi164 ceph-mon[49318]: Cluster is now healthy 2024-09-10T03:39:16.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:16 smithi164 ceph-mon[49318]: mds.? [v2:172.21.15.164:6832/1545713087,v1:172.21.15.164:6833/1545713087] up:boot 2024-09-10T03:39:16.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:16 smithi164 ceph-mon[49318]: fsmap foofs:1 {0=foofs.smithi130.ukcxnb=up:active} 1 up:standby 2024-09-10T03:39:16.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:16 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds metadata", "who": "foofs.smithi164.awiezo"} : dispatch 2024-09-10T03:39:16.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:16 smithi130 ceph-mon[63591]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-10T03:39:16.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:16 smithi130 ceph-mon[63591]: Cluster is now healthy 2024-09-10T03:39:16.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:16 smithi130 ceph-mon[63591]: mds.? [v2:172.21.15.164:6832/1545713087,v1:172.21.15.164:6833/1545713087] up:boot 2024-09-10T03:39:16.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:16 smithi130 ceph-mon[63591]: fsmap foofs:1 {0=foofs.smithi130.ukcxnb=up:active} 1 up:standby 2024-09-10T03:39:16.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:16 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "mds metadata", "who": "foofs.smithi164.awiezo"} : dispatch 2024-09-10T03:39:17.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:17 smithi130 ceph-mon[63591]: pgmap v322: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 12 KiB/s wr, 10 op/s 2024-09-10T03:39:17.541 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:17 smithi164 ceph-mon[49318]: pgmap v322: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 12 KiB/s wr, 10 op/s 2024-09-10T03:39:19.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:19 smithi164 ceph-mon[49318]: pgmap v323: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.7 KiB/s wr, 14 op/s 2024-09-10T03:39:19.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:19 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:19.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:19 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:19.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:19 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:19.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:19 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:19.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:18 smithi130 ceph-mon[63591]: pgmap v323: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.7 KiB/s wr, 14 op/s 2024-09-10T03:39:19.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:18 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:19.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:18 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:19.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:18 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:19.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:18 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:21.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:21 smithi164 ceph-mon[49318]: pgmap v324: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.6 KiB/s wr, 14 op/s 2024-09-10T03:39:21.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:21 smithi130 ceph-mon[63591]: pgmap v324: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.6 KiB/s wr, 14 op/s 2024-09-10T03:39:22.846 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: Detected new or changed devices on smithi164 2024-09-10T03:39:22.846 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:22.846 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:22.846 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:39:22.846 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:39:22.846 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:22.846 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:22.846 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: pgmap v325: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-09-10T03:39:22.846 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:39:22.847 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.847 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.847 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.847 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.847 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.847 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: Upgrade: Setting container_image for all mds 2024-09-10T03:39:22.847 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:22.847 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi130.ukcxnb"} : dispatch 2024-09-10T03:39:22.847 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi130.ukcxnb"}]': finished 2024-09-10T03:39:22.847 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi164.awiezo"} : dispatch 2024-09-10T03:39:22.847 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi164.awiezo"}]': finished 2024-09-10T03:39:22.847 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.847 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: Upgrade: Setting container_image for all rgw 2024-09-10T03:39:22.848 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:22.848 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.848 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: Upgrade: Setting container_image for all rbd-mirror 2024-09-10T03:39:22.848 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:22.848 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.851 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.851 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: Upgrade: Setting container_image for all ceph-exporter 2024-09-10T03:39:22.851 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:22.851 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.851 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: Upgrade: Setting container_image for all iscsi 2024-09-10T03:39:22.852 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:22 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:22.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: Detected new or changed devices on smithi164 2024-09-10T03:39:22.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:22.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:22.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:39:22.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:39:22.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:22.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:22.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: pgmap v325: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-09-10T03:39:22.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:39:22.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: Upgrade: Setting container_image for all mds 2024-09-10T03:39:22.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:22.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi130.ukcxnb"} : dispatch 2024-09-10T03:39:22.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi130.ukcxnb"}]': finished 2024-09-10T03:39:22.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi164.awiezo"} : dispatch 2024-09-10T03:39:22.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi164.awiezo"}]': finished 2024-09-10T03:39:22.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: Upgrade: Setting container_image for all rgw 2024-09-10T03:39:22.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:22.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: Upgrade: Setting container_image for all rbd-mirror 2024-09-10T03:39:22.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:22.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: Upgrade: Setting container_image for all ceph-exporter 2024-09-10T03:39:22.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:22.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:22.961 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: Upgrade: Setting container_image for all iscsi 2024-09-10T03:39:22.961 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:22 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:25.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:25 smithi130 ceph-mon[63591]: pgmap v326: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 7.2 KiB/s wr, 8 op/s 2024-09-10T03:39:25.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:25 smithi130 ceph-mon[63591]: Upgrade: Updating nfs.foo.0.0.smithi130.sanuoy (1/2) 2024-09-10T03:39:25.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:25 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:25.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:25 smithi130 ceph-mon[63591]: Creating key for client.nfs.foo.0.0.smithi130.sanuoy 2024-09-10T03:39:25.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:25 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi130.sanuoy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-10T03:39:25.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:25 smithi130 ceph-mon[63591]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-10T03:39:25.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:25 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-10T03:39:25.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:25 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-10T03:39:25.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:25 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:39:25.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:25 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-10T03:39:25.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:25 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-10T03:39:25.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:25 smithi130 ceph-mon[63591]: Rados config object exists: conf-nfs.foo 2024-09-10T03:39:25.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:25 smithi130 ceph-mon[63591]: Creating key for client.nfs.foo.0.0.smithi130.sanuoy-rgw 2024-09-10T03:39:25.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:25 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi130.sanuoy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-10T03:39:25.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:25 smithi130 ceph-mon[63591]: Bind address in nfs.foo.0.0.smithi130.sanuoy's ganesha conf is defaulting to empty 2024-09-10T03:39:25.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:25 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:39:25.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:25 smithi130 ceph-mon[63591]: Deploying daemon nfs.foo.0.0.smithi130.sanuoy on smithi130 2024-09-10T03:39:25.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:25 smithi164 ceph-mon[49318]: pgmap v326: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 7.2 KiB/s wr, 8 op/s 2024-09-10T03:39:25.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:25 smithi164 ceph-mon[49318]: Upgrade: Updating nfs.foo.0.0.smithi130.sanuoy (1/2) 2024-09-10T03:39:25.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:25 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:25.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:25 smithi164 ceph-mon[49318]: Creating key for client.nfs.foo.0.0.smithi130.sanuoy 2024-09-10T03:39:25.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:25 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi130.sanuoy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-10T03:39:25.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:25 smithi164 ceph-mon[49318]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-10T03:39:25.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:25 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-10T03:39:25.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:25 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-10T03:39:25.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:25 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:39:25.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:25 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-10T03:39:25.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:25 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-10T03:39:25.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:25 smithi164 ceph-mon[49318]: Rados config object exists: conf-nfs.foo 2024-09-10T03:39:25.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:25 smithi164 ceph-mon[49318]: Creating key for client.nfs.foo.0.0.smithi130.sanuoy-rgw 2024-09-10T03:39:25.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:25 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi130.sanuoy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-10T03:39:25.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:25 smithi164 ceph-mon[49318]: Bind address in nfs.foo.0.0.smithi130.sanuoy's ganesha conf is defaulting to empty 2024-09-10T03:39:25.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:25 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:39:25.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:25 smithi164 ceph-mon[49318]: Deploying daemon nfs.foo.0.0.smithi130.sanuoy on smithi130 2024-09-10T03:39:27.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:27 smithi164 ceph-mon[49318]: pgmap v327: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 363 KiB/s rd, 7.2 KiB/s wr, 6 op/s 2024-09-10T03:39:27.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:27 smithi130 ceph-mon[63591]: pgmap v327: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 363 KiB/s rd, 7.2 KiB/s wr, 6 op/s 2024-09-10T03:39:28.463 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:39:29.178 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:39:29.179 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (10m) 27s ago 19m 19.8M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:39:29.179 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (6m) 27s ago 19m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 8c7cfb7e87a1 2024-09-10T03:39:29.179 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (6m) 11s ago 18m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 243c094648b8 2024-09-10T03:39:29.179 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (10m) 27s ago 18m 57.6M - dad864ee21e9 72ba01ac1871 2024-09-10T03:39:29.179 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (33s) 27s ago 13m 11.0M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bda6b74a8b6a 2024-09-10T03:39:29.179 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (14s) 11s ago 13m 15.2M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 9c465297ee17 2024-09-10T03:39:29.179 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (9m) 27s ago 20m 523M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:39:29.179 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283,8765 running (8m) 11s ago 18m 458M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e4034e8e2a6d 2024-09-10T03:39:29.179 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (8m) 27s ago 20m 63.5M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 22e5cf56527a 2024-09-10T03:39:29.179 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (7m) 11s ago 18m 48.4M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 7d80bd0d15b9 2024-09-10T03:39:29.179 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 starting - - - - 2024-09-10T03:39:29.179 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (12m) 11s ago 12m 90.2M - 4.0 e1d6a67b021e 25c7483cb6c4 2024-09-10T03:39:29.179 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (9m) 27s ago 18m 10.2M - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:39:29.179 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (9m) 11s ago 18m 11.2M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:39:29.180 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (6m) 27s ago 17m 432M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b de3cbff0098b 2024-09-10T03:39:29.180 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (5m) 27s ago 17m 321M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f0722a6d1c90 2024-09-10T03:39:29.180 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (4m) 27s ago 16m 363M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b a871d2ccf96e 2024-09-10T03:39:29.180 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (3m) 27s ago 16m 385M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f2bc83e91fa2 2024-09-10T03:39:29.180 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (2m) 11s ago 15m 349M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 5e1e1b5e1266 2024-09-10T03:39:29.180 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (2m) 11s ago 15m 336M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 76ab1aa88b44 2024-09-10T03:39:29.180 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (98s) 11s ago 15m 287M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b ca90894c3878 2024-09-10T03:39:29.180 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (61s) 11s ago 14m 314M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 69f0162ef515 2024-09-10T03:39:29.180 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (8m) 27s ago 18m 56.7M - 2.51.0 1d3b7f56885b f59b02fac5b3 2024-09-10T03:39:29.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:29 smithi164 ceph-mon[49318]: pgmap v328: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 7.7 KiB/s wr, 7 op/s 2024-09-10T03:39:29.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:29 smithi164 ceph-mon[49318]: from='mds.? [v2:172.21.15.130:6834/340154171,v1:172.21.15.130:6835/340154171]' entity='mds.foofs.smithi130.ukcxnb' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.130:0/2159933080"} : dispatch 2024-09-10T03:39:29.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:29 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:29.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:29 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:29.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:29 smithi130 ceph-mon[63591]: pgmap v328: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 7.7 KiB/s wr, 7 op/s 2024-09-10T03:39:29.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:29 smithi130 ceph-mon[63591]: from='mds.? [v2:172.21.15.130:6834/340154171,v1:172.21.15.130:6835/340154171]' entity='mds.foofs.smithi130.ukcxnb' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.130:0/2159933080"} : dispatch 2024-09-10T03:39:29.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:29 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:29.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:29 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:29.610 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:39:29.610 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:39:29.611 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:39:29.611 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:39:29.611 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:39:29.611 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:39:29.611 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:39:29.611 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:39:29.611 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 8 2024-09-10T03:39:29.611 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:39:29.611 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:39:29.611 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:39:29.611 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:39:29.611 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:39:29.611 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 14 2024-09-10T03:39:29.611 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:39:29.612 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:39:29.971 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:39:29.971 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:39:29.971 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:39:29.971 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:39:29.971 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-09-10T03:39:29.971 INFO:teuthology.orchestra.run.smithi130.stdout: "mds", 2024-09-10T03:39:29.971 INFO:teuthology.orchestra.run.smithi130.stdout: "crash", 2024-09-10T03:39:29.972 INFO:teuthology.orchestra.run.smithi130.stdout: "mon", 2024-09-10T03:39:29.972 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr", 2024-09-10T03:39:29.972 INFO:teuthology.orchestra.run.smithi130.stdout: "osd" 2024-09-10T03:39:29.972 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-09-10T03:39:29.972 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "16/23 daemons upgraded", 2024-09-10T03:39:29.972 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading nfs daemons", 2024-09-10T03:39:29.972 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:39:29.972 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:39:30.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:30 smithi164 ceph-mon[49318]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:39:30.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:30 smithi164 ceph-mon[49318]: Evicting (and blocklisting) client session 14556 (172.21.15.130:0/2159933080) 2024-09-10T03:39:30.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:30 smithi164 ceph-mon[49318]: from='client.34516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:39:30.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:30 smithi164 ceph-mon[49318]: from='client.34532 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:39:30.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:30 smithi164 ceph-mon[49318]: from='mds.? [v2:172.21.15.130:6834/340154171,v1:172.21.15.130:6835/340154171]' entity='mds.foofs.smithi130.ukcxnb' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.130:0/2159933080"}]': finished 2024-09-10T03:39:30.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:30 smithi164 ceph-mon[49318]: osdmap e160: 8 total, 8 up, 8 in 2024-09-10T03:39:30.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:30 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/55365999' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:30.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:30.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.wfpoty", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-10T03:39:30.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-10T03:39:30.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-10T03:39:30.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:39:30.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-10T03:39:30.209 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-10T03:39:30.209 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:30.209 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:39:30.209 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.wfpoty-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-10T03:39:30.209 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:39:30.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:30 smithi130 ceph-mon[63591]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:39:30.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:30 smithi130 ceph-mon[63591]: Evicting (and blocklisting) client session 14556 (172.21.15.130:0/2159933080) 2024-09-10T03:39:30.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:30 smithi130 ceph-mon[63591]: from='client.34516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:39:30.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:30 smithi130 ceph-mon[63591]: from='client.34532 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:39:30.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:30 smithi130 ceph-mon[63591]: from='mds.? [v2:172.21.15.130:6834/340154171,v1:172.21.15.130:6835/340154171]' entity='mds.foofs.smithi130.ukcxnb' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.130:0/2159933080"}]': finished 2024-09-10T03:39:30.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:30 smithi130 ceph-mon[63591]: osdmap e160: 8 total, 8 up, 8 in 2024-09-10T03:39:30.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:30 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/55365999' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:30.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:30.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.wfpoty", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-10T03:39:30.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-10T03:39:30.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-10T03:39:30.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:39:30.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-10T03:39:30.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-10T03:39:30.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:30.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:39:30.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi164.wfpoty-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-10T03:39:30.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:39:30.407 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-09-10T03:39:31.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:31 smithi164 ceph-mon[49318]: pgmap v330: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 614 B/s wr, 1 op/s 2024-09-10T03:39:31.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:31 smithi164 ceph-mon[49318]: from='client.34540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:39:31.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:31 smithi164 ceph-mon[49318]: Upgrade: Updating nfs.foo.1.0.smithi164.wfpoty (2/2) 2024-09-10T03:39:31.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:31 smithi164 ceph-mon[49318]: Creating key for client.nfs.foo.1.0.smithi164.wfpoty 2024-09-10T03:39:31.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:31 smithi164 ceph-mon[49318]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-10T03:39:31.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:31 smithi164 ceph-mon[49318]: Rados config object exists: conf-nfs.foo 2024-09-10T03:39:31.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:31 smithi164 ceph-mon[49318]: Creating key for client.nfs.foo.1.0.smithi164.wfpoty-rgw 2024-09-10T03:39:31.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:31 smithi164 ceph-mon[49318]: Bind address in nfs.foo.1.0.smithi164.wfpoty's ganesha conf is defaulting to empty 2024-09-10T03:39:31.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:31 smithi164 ceph-mon[49318]: Deploying daemon nfs.foo.1.0.smithi164.wfpoty on smithi164 2024-09-10T03:39:31.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:31 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/1511624079' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:39:31.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:31 smithi130 ceph-mon[63591]: pgmap v330: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 614 B/s wr, 1 op/s 2024-09-10T03:39:31.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:31 smithi130 ceph-mon[63591]: from='client.34540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:39:31.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:31 smithi130 ceph-mon[63591]: Upgrade: Updating nfs.foo.1.0.smithi164.wfpoty (2/2) 2024-09-10T03:39:31.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:31 smithi130 ceph-mon[63591]: Creating key for client.nfs.foo.1.0.smithi164.wfpoty 2024-09-10T03:39:31.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:31 smithi130 ceph-mon[63591]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-10T03:39:31.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:31 smithi130 ceph-mon[63591]: Rados config object exists: conf-nfs.foo 2024-09-10T03:39:31.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:31 smithi130 ceph-mon[63591]: Creating key for client.nfs.foo.1.0.smithi164.wfpoty-rgw 2024-09-10T03:39:31.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:31 smithi130 ceph-mon[63591]: Bind address in nfs.foo.1.0.smithi164.wfpoty's ganesha conf is defaulting to empty 2024-09-10T03:39:31.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:31 smithi130 ceph-mon[63591]: Deploying daemon nfs.foo.1.0.smithi164.wfpoty on smithi164 2024-09-10T03:39:31.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:31 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/1511624079' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:39:33.306 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:33 smithi164 ceph-mon[49318]: pgmap v331: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 614 B/s wr, 1 op/s 2024-09-10T03:39:33.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:33 smithi130 ceph-mon[63591]: pgmap v331: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 614 B/s wr, 1 op/s 2024-09-10T03:39:34.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:34 smithi130 ceph-mon[63591]: from='mds.? [v2:172.21.15.130:6834/340154171,v1:172.21.15.130:6835/340154171]' entity='mds.foofs.smithi130.ukcxnb' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.164:0/3206963038"} : dispatch 2024-09-10T03:39:34.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:34 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:34.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:34 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:34.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:34 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:39:34.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:34 smithi164 ceph-mon[49318]: from='mds.? [v2:172.21.15.130:6834/340154171,v1:172.21.15.130:6835/340154171]' entity='mds.foofs.smithi130.ukcxnb' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.164:0/3206963038"} : dispatch 2024-09-10T03:39:34.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:34 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:34.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:34 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:34.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:34 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:39:35.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:35 smithi164 ceph-mon[49318]: Evicting (and blocklisting) client session 24343 (172.21.15.164:0/3206963038) 2024-09-10T03:39:35.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:35 smithi164 ceph-mon[49318]: pgmap v332: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-10T03:39:35.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:35 smithi164 ceph-mon[49318]: from='mds.? [v2:172.21.15.130:6834/340154171,v1:172.21.15.130:6835/340154171]' entity='mds.foofs.smithi130.ukcxnb' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.164:0/3206963038"}]': finished 2024-09-10T03:39:35.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:35 smithi164 ceph-mon[49318]: osdmap e161: 8 total, 8 up, 8 in 2024-09-10T03:39:35.484 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:35 smithi130 ceph-mon[63591]: Evicting (and blocklisting) client session 24343 (172.21.15.164:0/3206963038) 2024-09-10T03:39:35.484 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:35 smithi130 ceph-mon[63591]: pgmap v332: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-10T03:39:35.484 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:35 smithi130 ceph-mon[63591]: from='mds.? [v2:172.21.15.130:6834/340154171,v1:172.21.15.130:6835/340154171]' entity='mds.foofs.smithi130.ukcxnb' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.164:0/3206963038"}]': finished 2024-09-10T03:39:35.485 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:35 smithi130 ceph-mon[63591]: osdmap e161: 8 total, 8 up, 8 in 2024-09-10T03:39:37.559 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:37 smithi164 ceph-mon[49318]: pgmap v334: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-09-10T03:39:37.559 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:37.559 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:37.559 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:37.559 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:37 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:37.639 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:37 smithi130 ceph-mon[63591]: pgmap v334: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-09-10T03:39:37.639 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:37.639 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:37.639 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:37.639 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:37 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:39.653 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:39 smithi164 ceph-mon[49318]: pgmap v335: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.6 KiB/s rd, 1.9 KiB/s wr, 6 op/s 2024-09-10T03:39:39.654 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:39 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:39.654 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:39 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:39.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:39 smithi130 ceph-mon[63591]: pgmap v335: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.6 KiB/s rd, 1.9 KiB/s wr, 6 op/s 2024-09-10T03:39:39.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:39 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:39.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:39 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:40.957 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:40 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:40.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:40 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:40.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:40 smithi164 ceph-mon[49318]: pgmap v336: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-09-10T03:39:40.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:40 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:40.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:40 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:41.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:40 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:41.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:40 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:41.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:40 smithi130 ceph-mon[63591]: pgmap v336: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-09-10T03:39:41.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:40 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:41.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:40 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:43.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:42 smithi130 ceph-mon[63591]: pgmap v337: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-09-10T03:39:43.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:42 smithi164 ceph-mon[49318]: pgmap v337: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-09-10T03:39:44.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:44.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:44.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:39:44.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:39:44.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:44.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:44.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:39:44.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.265 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: Upgrade: Setting container_image for all nfs 2024-09-10T03:39:44.265 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:44.265 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi130.sanuoy"} : dispatch 2024-09-10T03:39:44.265 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi130.sanuoy"}]': finished 2024-09-10T03:39:44.265 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi164.wfpoty"} : dispatch 2024-09-10T03:39:44.265 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi164.wfpoty"}]': finished 2024-09-10T03:39:44.265 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.265 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: Upgrade: Setting container_image for all nvmeof 2024-09-10T03:39:44.265 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:44.265 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.265 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.265 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:44 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:44.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:44.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:39:44.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:39:44.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:44.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:44.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:39:44.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: Upgrade: Setting container_image for all nfs 2024-09-10T03:39:44.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:44.459 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi130.sanuoy"} : dispatch 2024-09-10T03:39:44.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi130.sanuoy"}]': finished 2024-09-10T03:39:44.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi164.wfpoty"} : dispatch 2024-09-10T03:39:44.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi164.wfpoty"}]': finished 2024-09-10T03:39:44.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: Upgrade: Setting container_image for all nvmeof 2024-09-10T03:39:44.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:44.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:44.460 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:44 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:39:45.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:45 smithi164 ceph-mon[49318]: pgmap v338: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 819 B/s wr, 2 op/s 2024-09-10T03:39:45.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:45 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:39:45.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:45 smithi130 ceph-mon[63591]: pgmap v338: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 819 B/s wr, 2 op/s 2024-09-10T03:39:45.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:45 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:39:46.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:46 smithi164 ceph-mon[49318]: Upgrade: Updating grafana.smithi130 2024-09-10T03:39:46.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:46 smithi164 ceph-mon[49318]: Deploying daemon grafana.smithi130 on smithi130 2024-09-10T03:39:46.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:46 smithi130 ceph-mon[63591]: Upgrade: Updating grafana.smithi130 2024-09-10T03:39:46.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:46 smithi130 ceph-mon[63591]: Deploying daemon grafana.smithi130 on smithi130 2024-09-10T03:39:47.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:47 smithi164 ceph-mon[49318]: pgmap v339: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s rd, 704 B/s wr, 2 op/s 2024-09-10T03:39:47.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:47 smithi130 ceph-mon[63591]: pgmap v339: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s rd, 704 B/s wr, 2 op/s 2024-09-10T03:39:48.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:48 smithi164 ceph-mon[49318]: pgmap v340: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 1023 B/s wr, 3 op/s 2024-09-10T03:39:48.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:48 smithi130 ceph-mon[63591]: pgmap v340: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 1023 B/s wr, 3 op/s 2024-09-10T03:39:51.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:50 smithi164 ceph-mon[49318]: pgmap v341: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 426 B/s wr, 1 op/s 2024-09-10T03:39:51.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:50 smithi130 ceph-mon[63591]: pgmap v341: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 426 B/s wr, 1 op/s 2024-09-10T03:39:53.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:52 smithi164 ceph-mon[49318]: pgmap v342: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-10T03:39:53.232 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:52 smithi130 ceph-mon[63591]: pgmap v342: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-10T03:39:55.098 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:54 smithi130 ceph-mon[63591]: pgmap v343: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-10T03:39:55.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:54 smithi164 ceph-mon[49318]: pgmap v343: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-10T03:39:57.066 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:56 smithi130 ceph-mon[63591]: pgmap v344: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-10T03:39:57.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:56 smithi164 ceph-mon[49318]: pgmap v344: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-10T03:39:58.160 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:57 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:58.160 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:57 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:58.160 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:57 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:39:58.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:57 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:58.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:57 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:39:58.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:57 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:39:59.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:39:58 smithi130 ceph-mon[63591]: pgmap v345: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 853 B/s rd, 341 B/s wr, 1 op/s 2024-09-10T03:39:59.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:39:58 smithi164 ceph-mon[49318]: pgmap v345: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 853 B/s rd, 341 B/s wr, 1 op/s 2024-09-10T03:40:00.796 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:40:01.104 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:00 smithi130 ceph-mon[63591]: pgmap v346: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:01.104 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:00 smithi130 ceph-mon[63591]: overall HEALTH_OK 2024-09-10T03:40:01.104 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:40:01.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:00 smithi164 ceph-mon[49318]: pgmap v346: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:00 smithi164 ceph-mon[49318]: overall HEALTH_OK 2024-09-10T03:40:01.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:40:01.573 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:40:01.573 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (10m) 22s ago 19m 19.9M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:40:01.573 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (7m) 22s ago 19m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 8c7cfb7e87a1 2024-09-10T03:40:01.573 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (7m) 25s ago 19m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 243c094648b8 2024-09-10T03:40:01.573 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 starting - - - - 2024-09-10T03:40:01.573 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (66s) 22s ago 13m 30.0M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bda6b74a8b6a 2024-09-10T03:40:01.573 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (46s) 25s ago 13m 15.3M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 9c465297ee17 2024-09-10T03:40:01.573 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (9m) 22s ago 20m 524M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:40:01.574 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283,8765 running (9m) 25s ago 19m 458M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e4034e8e2a6d 2024-09-10T03:40:01.574 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (8m) 22s ago 21m 65.2M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 22e5cf56527a 2024-09-10T03:40:01.574 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (8m) 25s ago 19m 49.1M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 7d80bd0d15b9 2024-09-10T03:40:01.574 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (33s) 22s ago 13m 62.4M - 5.9 d6d839bc8a4b 21eda0e941a0 2024-09-10T03:40:01.574 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (28s) 25s ago 13m 26.0M - 5.9 d6d839bc8a4b 90b94699ad43 2024-09-10T03:40:01.574 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (10m) 22s ago 19m 10.4M - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:40:01.574 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (10m) 25s ago 18m 11.2M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:40:01.574 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (6m) 22s ago 18m 433M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b de3cbff0098b 2024-09-10T03:40:01.574 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (5m) 22s ago 17m 321M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f0722a6d1c90 2024-09-10T03:40:01.574 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (4m) 22s ago 17m 364M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b a871d2ccf96e 2024-09-10T03:40:01.574 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (4m) 22s ago 16m 388M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f2bc83e91fa2 2024-09-10T03:40:01.574 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (3m) 25s ago 16m 348M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 5e1e1b5e1266 2024-09-10T03:40:01.574 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (2m) 25s ago 16m 337M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 76ab1aa88b44 2024-09-10T03:40:01.575 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (2m) 25s ago 15m 286M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b ca90894c3878 2024-09-10T03:40:01.575 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (94s) 25s ago 15m 314M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 69f0162ef515 2024-09-10T03:40:01.575 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (9m) 22s ago 18m 56.7M - 2.51.0 1d3b7f56885b f59b02fac5b3 2024-09-10T03:40:02.006 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:40:02.006 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:40:02.006 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:40:02.006 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:40:02.006 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:40:02.006 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:40:02.007 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:40:02.007 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:40:02.007 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 8 2024-09-10T03:40:02.007 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:40:02.007 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:40:02.007 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:40:02.007 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:40:02.007 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:40:02.007 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 14 2024-09-10T03:40:02.007 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:40:02.007 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:40:02.107 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:02 smithi130 ceph-mon[63591]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:40:02.108 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:02 smithi130 ceph-mon[63591]: from='client.34574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:40:02.364 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:40:02.364 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", 2024-09-10T03:40:02.364 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": true, 2024-09-10T03:40:02.364 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-10T03:40:02.364 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [ 2024-09-10T03:40:02.365 INFO:teuthology.orchestra.run.smithi130.stdout: "nfs", 2024-09-10T03:40:02.365 INFO:teuthology.orchestra.run.smithi130.stdout: "mds", 2024-09-10T03:40:02.365 INFO:teuthology.orchestra.run.smithi130.stdout: "crash", 2024-09-10T03:40:02.365 INFO:teuthology.orchestra.run.smithi130.stdout: "mon", 2024-09-10T03:40:02.365 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr", 2024-09-10T03:40:02.365 INFO:teuthology.orchestra.run.smithi130.stdout: "osd" 2024-09-10T03:40:02.365 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-09-10T03:40:02.365 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": "18/23 daemons upgraded", 2024-09-10T03:40:02.365 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "Currently upgrading grafana daemons", 2024-09-10T03:40:02.365 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:40:02.365 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:40:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:02 smithi164 ceph-mon[49318]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:40:02.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:02 smithi164 ceph-mon[49318]: from='client.34574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:40:02.800 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-09-10T03:40:03.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:03 smithi130 ceph-mon[63591]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:40:03.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:03 smithi130 ceph-mon[63591]: pgmap v347: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:03.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:03 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/3429101610' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:03.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:03 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:03.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:03 smithi130 ceph-mon[63591]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:40:03.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:03 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:03.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:03 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/3736007273' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:40:03.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:03 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:03.264 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:03 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:03 smithi164 ceph-mon[49318]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:40:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:03 smithi164 ceph-mon[49318]: pgmap v347: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:03 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/3429101610' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:03 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:03 smithi164 ceph-mon[49318]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:40:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:03 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:03 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/3736007273' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:40:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:03 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:03.458 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:03 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:05.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:05 smithi130 ceph-mon[63591]: pgmap v348: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:05.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:05 smithi164 ceph-mon[49318]: pgmap v348: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:07.437 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:07 smithi164 ceph-mon[49318]: pgmap v349: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:07.515 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:07 smithi130 ceph-mon[63591]: pgmap v349: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:09.457 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:09 smithi164 ceph-mon[49318]: pgmap v350: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:40:09.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:09 smithi130 ceph-mon[63591]: pgmap v350: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:40:10.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:10.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:10.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:40:10.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:40:10.958 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:10.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:10.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-10T03:40:10.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-10T03:40:10.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: pgmap v351: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:10.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:40:10.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:10.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:10.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:10.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:10.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:10.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:10.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:10.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:10.959 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:10.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:10.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:10.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:10.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:10.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:10.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:10.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:10.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:10.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:10.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: Upgrade: Finalizing container_image settings 2024-09-10T03:40:10.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:10.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-10T03:40:10.960 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-10T03:40:10.961 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-10T03:40:10.961 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-10T03:40:10.961 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-10T03:40:10.961 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-10T03:40:10.961 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-10T03:40:10.961 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-10T03:40:10.961 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-10T03:40:10.961 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-10T03:40:10.961 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-10T03:40:10.961 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-10T03:40:10.961 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-10T03:40:10.961 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-10T03:40:10.961 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-10T03:40:10.962 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-10T03:40:10.962 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-10T03:40:10.962 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-10T03:40:10.962 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-10T03:40:10.962 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-10T03:40:10.962 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-10T03:40:10.962 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-10T03:40:10.962 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-10T03:40:10.963 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-10T03:40:10.963 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-10T03:40:10.963 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-10T03:40:10.963 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-10T03:40:10.963 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-10T03:40:10.963 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-10T03:40:10.963 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: Upgrade: Complete! 2024-09-10T03:40:10.963 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-10T03:40:10.963 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-10T03:40:10.963 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:40:10.963 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:10.963 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:10.963 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:10.963 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:40:10.964 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:40:10.964 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:10.964 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:10.964 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:40:10.964 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:40:10.964 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:40:10.964 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:10.964 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:10 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:11.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:11.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:11.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:40:11.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:40:11.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:11.013 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:11.014 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-10T03:40:11.014 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-10T03:40:11.014 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: pgmap v351: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:11.014 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:40:11.014 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:11.014 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:11.014 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:11.014 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:11.014 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:11.014 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:11.014 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:11.014 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:11.014 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:11.014 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:11.015 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:11.015 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:11.015 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:11.015 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:11.015 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:11.015 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:11.015 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:11.015 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:11.015 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: Upgrade: Finalizing container_image settings 2024-09-10T03:40:11.015 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:11.015 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-10T03:40:11.015 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-10T03:40:11.015 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-10T03:40:11.016 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-10T03:40:11.016 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-10T03:40:11.016 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-10T03:40:11.016 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-10T03:40:11.016 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-10T03:40:11.016 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-10T03:40:11.016 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-10T03:40:11.016 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-10T03:40:11.016 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-10T03:40:11.016 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-10T03:40:11.016 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-10T03:40:11.016 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-10T03:40:11.016 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-10T03:40:11.016 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-10T03:40:11.017 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-10T03:40:11.017 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-10T03:40:11.017 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-10T03:40:11.017 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-10T03:40:11.017 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-10T03:40:11.017 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-10T03:40:11.017 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-10T03:40:11.018 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-10T03:40:11.018 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-10T03:40:11.018 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-10T03:40:11.018 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-10T03:40:11.018 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-10T03:40:11.018 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: Upgrade: Complete! 2024-09-10T03:40:11.018 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-10T03:40:11.018 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-10T03:40:11.018 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:40:11.018 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:11.018 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:11.018 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:11.018 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:40:11.018 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:40:11.019 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:11.019 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:11.019 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:40:11.019 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:40:11.019 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:40:11.019 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:11.019 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:10 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:40:13.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:12 smithi164 ceph-mon[49318]: pgmap v352: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:13.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:12 smithi130 ceph-mon[63591]: pgmap v352: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:15.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:14 smithi164 ceph-mon[49318]: pgmap v353: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:15.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:14 smithi130 ceph-mon[63591]: pgmap v353: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:16.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:15 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:40:16.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:15 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:40:17.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:16 smithi164 ceph-mon[49318]: pgmap v354: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:17.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:16 smithi130 ceph-mon[63591]: pgmap v354: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:19.209 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:18 smithi164 ceph-mon[49318]: pgmap v355: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:40:19.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:18 smithi130 ceph-mon[63591]: pgmap v355: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:40:21.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:20 smithi164 ceph-mon[49318]: pgmap v356: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:21.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:20 smithi130 ceph-mon[63591]: pgmap v356: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:23.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:22 smithi164 ceph-mon[49318]: pgmap v357: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:23.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:22 smithi130 ceph-mon[63591]: pgmap v357: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:25.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:24 smithi164 ceph-mon[49318]: pgmap v358: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:25.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:24 smithi130 ceph-mon[63591]: pgmap v358: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:27.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:26 smithi164 ceph-mon[49318]: pgmap v359: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:27.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:26 smithi130 ceph-mon[63591]: pgmap v359: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:29.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:28 smithi164 ceph-mon[49318]: pgmap v360: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:40:29.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:28 smithi130 ceph-mon[63591]: pgmap v360: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:40:31.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:30 smithi164 ceph-mon[49318]: pgmap v361: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:31.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:40:31.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:30 smithi130 ceph-mon[63591]: pgmap v361: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:31.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:40:33.170 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:32 smithi130 ceph-mon[63591]: pgmap v362: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:33.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:32 smithi164 ceph-mon[49318]: pgmap v362: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:33.909 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -e sha1=3a5e3f16720f432aea965c40a8f6eb480095ec6c -- bash -c 'ceph orch ps' 2024-09-10T03:40:34.207 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:33 smithi130 ceph-mon[63591]: from='client.34594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:40:34.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:33 smithi164 ceph-mon[49318]: from='client.34594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:40:35.183 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:34 smithi130 ceph-mon[63591]: pgmap v363: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:35.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:34 smithi164 ceph-mon[49318]: pgmap v363: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:35.927 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:40:35.927 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (11m) 33s ago 20m 19.9M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:40:35.927 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (7m) 33s ago 20m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 8c7cfb7e87a1 2024-09-10T03:40:35.928 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (7m) 59s ago 19m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 243c094648b8 2024-09-10T03:40:35.928 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (39s) 33s ago 20m 34.9M - 10.4.0 c8b91775d855 a7179b233516 2024-09-10T03:40:35.928 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (100s) 33s ago 14m 30.2M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bda6b74a8b6a 2024-09-10T03:40:35.928 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (81s) 59s ago 14m 15.3M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 9c465297ee17 2024-09-10T03:40:35.928 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (10m) 33s ago 21m 533M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:40:35.928 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283,8765 running (9m) 59s ago 19m 458M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e4034e8e2a6d 2024-09-10T03:40:35.928 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (9m) 33s ago 21m 66.3M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 22e5cf56527a 2024-09-10T03:40:35.928 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (8m) 59s ago 19m 49.1M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 7d80bd0d15b9 2024-09-10T03:40:35.928 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (67s) 33s ago 13m 63.1M - 5.9 d6d839bc8a4b 21eda0e941a0 2024-09-10T03:40:35.928 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (62s) 59s ago 13m 26.0M - 5.9 d6d839bc8a4b 90b94699ad43 2024-09-10T03:40:35.928 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (11m) 33s ago 20m 11.0M - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:40:35.928 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (10m) 59s ago 19m 11.2M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:40:35.928 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (7m) 33s ago 18m 433M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b de3cbff0098b 2024-09-10T03:40:35.929 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (6m) 33s ago 18m 323M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f0722a6d1c90 2024-09-10T03:40:35.929 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (5m) 33s ago 17m 365M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b a871d2ccf96e 2024-09-10T03:40:35.929 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (4m) 33s ago 17m 389M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f2bc83e91fa2 2024-09-10T03:40:35.929 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (3m) 59s ago 16m 348M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 5e1e1b5e1266 2024-09-10T03:40:35.929 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (3m) 59s ago 16m 337M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 76ab1aa88b44 2024-09-10T03:40:35.929 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (2m) 59s ago 16m 286M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b ca90894c3878 2024-09-10T03:40:35.929 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (2m) 59s ago 15m 314M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 69f0162ef515 2024-09-10T03:40:35.929 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (9m) 33s ago 19m 56.7M - 2.51.0 1d3b7f56885b f59b02fac5b3 2024-09-10T03:40:36.522 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -e sha1=3a5e3f16720f432aea965c40a8f6eb480095ec6c -- bash -c 'ceph versions' 2024-09-10T03:40:37.159 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:36 smithi130 ceph-mon[63591]: pgmap v364: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:37.159 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:36 smithi130 ceph-mon[63591]: from='client.34598 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:40:37.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:36 smithi164 ceph-mon[49318]: pgmap v364: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:37.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:36 smithi164 ceph-mon[49318]: from='client.34598 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:40:38.571 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:40:38.571 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:40:38.571 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:40:38.571 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:40:38.571 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:40:38.571 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:40:38.571 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:40:38.571 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:40:38.571 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 8 2024-09-10T03:40:38.571 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:40:38.571 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:40:38.571 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:40:38.572 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:40:38.572 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:40:38.572 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 14 2024-09-10T03:40:38.572 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:40:38.572 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:40:39.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:38 smithi164 ceph-mon[49318]: pgmap v365: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:40:39.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:38 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/2102111715' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:39.220 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:38 smithi130 ceph-mon[63591]: pgmap v365: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:40:39.220 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:38 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/2102111715' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:40:39.245 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -e sha1=3a5e3f16720f432aea965c40a8f6eb480095ec6c -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-10T03:40:41.155 INFO:teuthology.orchestra.run.smithi130.stdout:wait for servicemap items w/ changing names to refresh 2024-09-10T03:40:41.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:40 smithi164 ceph-mon[49318]: pgmap v366: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:41.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:40 smithi130 ceph-mon[63591]: pgmap v366: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:41.843 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -e sha1=3a5e3f16720f432aea965c40a8f6eb480095ec6c -- bash -c 'sleep 60' 2024-09-10T03:40:43.182 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:42 smithi130 ceph-mon[63591]: pgmap v367: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:43.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:42 smithi164 ceph-mon[49318]: pgmap v367: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:45.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:44 smithi164 ceph-mon[49318]: pgmap v368: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:45.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:44 smithi130 ceph-mon[63591]: pgmap v368: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:46.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:45 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:40:46.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:45 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:40:47.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:46 smithi164 ceph-mon[49318]: pgmap v369: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:47.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:46 smithi130 ceph-mon[63591]: pgmap v369: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:49.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:48 smithi164 ceph-mon[49318]: pgmap v370: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:40:49.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:48 smithi130 ceph-mon[63591]: pgmap v370: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:40:51.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:50 smithi164 ceph-mon[49318]: pgmap v371: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:51.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:50 smithi130 ceph-mon[63591]: pgmap v371: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:53.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:52 smithi164 ceph-mon[49318]: pgmap v372: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:53.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:52 smithi130 ceph-mon[63591]: pgmap v372: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:55.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:54 smithi164 ceph-mon[49318]: pgmap v373: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:55.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:54 smithi130 ceph-mon[63591]: pgmap v373: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:57.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:57 smithi164 ceph-mon[49318]: pgmap v374: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:57.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:57 smithi130 ceph-mon[63591]: pgmap v374: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:40:59.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:40:59 smithi164 ceph-mon[49318]: pgmap v375: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:40:59.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:40:59 smithi130 ceph-mon[63591]: pgmap v375: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:41:00.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:41:00.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:41:01.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:01 smithi164 ceph-mon[49318]: pgmap v376: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:01.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:01 smithi130 ceph-mon[63591]: pgmap v376: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:03.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:03 smithi164 ceph-mon[49318]: pgmap v377: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:03.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:03 smithi130 ceph-mon[63591]: pgmap v377: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:05.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:05 smithi164 ceph-mon[49318]: pgmap v378: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:05.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:05 smithi130 ceph-mon[63591]: pgmap v378: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:07.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:07 smithi164 ceph-mon[49318]: pgmap v379: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:07.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:07 smithi130 ceph-mon[63591]: pgmap v379: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:09.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:09 smithi164 ceph-mon[49318]: pgmap v380: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:41:09.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:09 smithi130 ceph-mon[63591]: pgmap v380: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:41:11.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:11 smithi164 ceph-mon[49318]: pgmap v381: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:11.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:11 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:41:11.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:11 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:41:11.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:11 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:41:11.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:11 smithi130 ceph-mon[63591]: pgmap v381: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:11.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:11 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:41:11.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:11 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:41:11.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:11 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:41:12.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:12 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:41:12.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:12 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:41:12.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:12 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:41:12.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:12 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:41:12.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:12 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:41:12.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:12 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:41:12.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:12 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:41:12.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:12 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:41:13.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:13 smithi164 ceph-mon[49318]: pgmap v382: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:13.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:13 smithi130 ceph-mon[63591]: pgmap v382: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:15.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:15 smithi164 ceph-mon[49318]: pgmap v383: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:15.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:15 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:41:15.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:15 smithi130 ceph-mon[63591]: pgmap v383: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:15.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:15 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:41:17.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:17 smithi164 ceph-mon[49318]: pgmap v384: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:17.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:17 smithi130 ceph-mon[63591]: pgmap v384: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:19.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:19 smithi164 ceph-mon[49318]: pgmap v385: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:41:19.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:19 smithi130 ceph-mon[63591]: pgmap v385: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:41:21.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:21 smithi164 ceph-mon[49318]: pgmap v386: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:21.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:21 smithi130 ceph-mon[63591]: pgmap v386: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:23.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:23 smithi164 ceph-mon[49318]: pgmap v387: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:23.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:23 smithi130 ceph-mon[63591]: pgmap v387: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:25.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:25 smithi164 ceph-mon[49318]: pgmap v388: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:25.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:25 smithi130 ceph-mon[63591]: pgmap v388: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:27.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:27 smithi164 ceph-mon[49318]: pgmap v389: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:27.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:27 smithi130 ceph-mon[63591]: pgmap v389: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:29.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:29 smithi164 ceph-mon[49318]: pgmap v390: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:41:29.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:29 smithi130 ceph-mon[63591]: pgmap v390: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:41:30.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:30 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:41:30.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:30 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:41:31.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:31 smithi164 ceph-mon[49318]: pgmap v391: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:31.764 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:31 smithi130 ceph-mon[63591]: pgmap v391: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:33.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:33 smithi164 ceph-mon[49318]: pgmap v392: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:33.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:33 smithi130 ceph-mon[63591]: pgmap v392: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:35.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:35 smithi164 ceph-mon[49318]: pgmap v393: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:35.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:35 smithi130 ceph-mon[63591]: pgmap v393: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:37.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:37 smithi164 ceph-mon[49318]: pgmap v394: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:37.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:37 smithi130 ceph-mon[63591]: pgmap v394: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:39.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:39 smithi164 ceph-mon[49318]: pgmap v395: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:41:39.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:39 smithi130 ceph-mon[63591]: pgmap v395: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:41:41.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:41 smithi164 ceph-mon[49318]: pgmap v396: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:41.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:41 smithi130 ceph-mon[63591]: pgmap v396: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:43.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:43 smithi164 ceph-mon[49318]: pgmap v397: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:43.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:43 smithi130 ceph-mon[63591]: pgmap v397: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:44.380 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -e sha1=3a5e3f16720f432aea965c40a8f6eb480095ec6c -- bash -c 'ceph orch ps' 2024-09-10T03:41:45.691 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:45 smithi130 ceph-mon[63591]: pgmap v398: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:45.691 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:45 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:41:45.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:45 smithi164 ceph-mon[49318]: pgmap v398: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:45.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:45 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:41:46.458 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-10T03:41:46.458 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.smithi130 smithi130 *:9093,9094 running (12m) 104s ago 21m 19.9M - 0.27.0 11f11916f8cd 71bde96f1284 2024-09-10T03:41:46.458 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi130 smithi130 running (9m) 104s ago 21m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 8c7cfb7e87a1 2024-09-10T03:41:46.458 INFO:teuthology.orchestra.run.smithi130.stdout:crash.smithi164 smithi164 running (9m) 2m ago 20m 6895k - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 243c094648b8 2024-09-10T03:41:46.458 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.smithi130 smithi130 *:3000 running (110s) 104s ago 21m 34.9M - 10.4.0 c8b91775d855 a7179b233516 2024-09-10T03:41:46.458 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi130.ukcxnb smithi130 running (2m) 104s ago 15m 30.2M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bda6b74a8b6a 2024-09-10T03:41:46.459 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foofs.smithi164.awiezo smithi164 running (2m) 2m ago 15m 15.3M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 9c465297ee17 2024-09-10T03:41:46.459 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi130.axcmel smithi130 *:8443,9283,8765 running (11m) 104s ago 22m 533M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b bf56417d3b59 2024-09-10T03:41:46.459 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.smithi164.bkfwap smithi164 *:8443,9283,8765 running (10m) 2m ago 20m 458M - 19.3.0-4885-g3a5e3f16 d6d839bc8a4b e4034e8e2a6d 2024-09-10T03:41:46.459 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi130 smithi130 running (10m) 104s ago 22m 66.3M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 22e5cf56527a 2024-09-10T03:41:46.459 INFO:teuthology.orchestra.run.smithi130.stdout:mon.smithi164 smithi164 running (9m) 2m ago 20m 49.1M 2048M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 7d80bd0d15b9 2024-09-10T03:41:46.459 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.0.0.smithi130.sanuoy smithi130 *:2049 running (2m) 104s ago 15m 63.1M - 5.9 d6d839bc8a4b 21eda0e941a0 2024-09-10T03:41:46.459 INFO:teuthology.orchestra.run.smithi130.stdout:nfs.foo.1.0.smithi164.wfpoty smithi164 *:2049 running (2m) 2m ago 15m 26.0M - 5.9 d6d839bc8a4b 90b94699ad43 2024-09-10T03:41:46.459 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi130 smithi130 *:9100 running (12m) 104s ago 21m 11.0M - 1.7.0 72c9c2088986 29dab42cf1ce 2024-09-10T03:41:46.459 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.smithi164 smithi164 *:9100 running (11m) 2m ago 20m 11.2M - 1.7.0 72c9c2088986 4dea306ad87d 2024-09-10T03:41:46.459 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (8m) 104s ago 19m 433M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b de3cbff0098b 2024-09-10T03:41:46.459 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (7m) 104s ago 19m 323M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f0722a6d1c90 2024-09-10T03:41:46.459 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (6m) 104s ago 18m 365M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b a871d2ccf96e 2024-09-10T03:41:46.459 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (5m) 104s ago 18m 389M 1936M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b f2bc83e91fa2 2024-09-10T03:41:46.459 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi164 running (5m) 2m ago 18m 348M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 5e1e1b5e1266 2024-09-10T03:41:46.460 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi164 running (4m) 2m ago 17m 337M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 76ab1aa88b44 2024-09-10T03:41:46.460 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi164 running (3m) 2m ago 17m 286M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b ca90894c3878 2024-09-10T03:41:46.460 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi164 running (3m) 2m ago 16m 314M 2704M 19.3.0-4885-g3a5e3f16 d6d839bc8a4b 69f0162ef515 2024-09-10T03:41:46.460 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.smithi130 smithi130 *:9095 running (10m) 104s ago 20m 56.7M - 2.51.0 1d3b7f56885b f59b02fac5b3 2024-09-10T03:41:47.149 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -e sha1=3a5e3f16720f432aea965c40a8f6eb480095ec6c -- bash -c 'ceph orch upgrade status' 2024-09-10T03:41:47.613 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:47 smithi130 ceph-mon[63591]: pgmap v399: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:47.709 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:47 smithi164 ceph-mon[49318]: pgmap v399: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:48.454 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:48 smithi130 ceph-mon[63591]: from='client.34606 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:41:48.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:48 smithi164 ceph-mon[49318]: from='client.34606 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:41:49.184 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:41:49.185 INFO:teuthology.orchestra.run.smithi130.stdout: "target_image": null, 2024-09-10T03:41:49.185 INFO:teuthology.orchestra.run.smithi130.stdout: "in_progress": false, 2024-09-10T03:41:49.185 INFO:teuthology.orchestra.run.smithi130.stdout: "which": "", 2024-09-10T03:41:49.185 INFO:teuthology.orchestra.run.smithi130.stdout: "services_complete": [], 2024-09-10T03:41:49.185 INFO:teuthology.orchestra.run.smithi130.stdout: "progress": null, 2024-09-10T03:41:49.185 INFO:teuthology.orchestra.run.smithi130.stdout: "message": "", 2024-09-10T03:41:49.185 INFO:teuthology.orchestra.run.smithi130.stdout: "is_paused": false 2024-09-10T03:41:49.185 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:41:49.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:49 smithi130 ceph-mon[63591]: pgmap v400: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:41:49.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:49 smithi164 ceph-mon[49318]: pgmap v400: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:41:49.844 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -e sha1=3a5e3f16720f432aea965c40a8f6eb480095ec6c -- bash -c 'ceph health detail' 2024-09-10T03:41:50.425 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:50 smithi130 ceph-mon[63591]: from='client.34610 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:41:50.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:50 smithi164 ceph-mon[49318]: from='client.34610 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:41:51.437 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:51 smithi130 ceph-mon[63591]: pgmap v401: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:51.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:51 smithi164 ceph-mon[49318]: pgmap v401: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:51.954 INFO:teuthology.orchestra.run.smithi130.stdout:HEALTH_OK 2024-09-10T03:41:52.592 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:52 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/1577856885' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:41:52.615 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -e sha1=3a5e3f16720f432aea965c40a8f6eb480095ec6c -- bash -c 'ceph versions' 2024-09-10T03:41:52.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:52 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/1577856885' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-10T03:41:53.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:53 smithi130 ceph-mon[63591]: pgmap v402: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:53.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:53 smithi164 ceph-mon[49318]: pgmap v402: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:54.638 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-09-10T03:41:54.639 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-09-10T03:41:54.639 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:41:54.639 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:41:54.639 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-09-10T03:41:54.639 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:41:54.639 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:41:54.639 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-09-10T03:41:54.639 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 8 2024-09-10T03:41:54.639 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:41:54.639 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-09-10T03:41:54.639 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 2 2024-09-10T03:41:54.639 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:41:54.640 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-09-10T03:41:54.640 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)": 14 2024-09-10T03:41:54.640 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:41:54.640 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-09-10T03:41:55.248 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -e sha1=3a5e3f16720f432aea965c40a8f6eb480095ec6c -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-10T03:41:55.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:55 smithi130 ceph-mon[63591]: pgmap v403: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:55.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:55 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/1220507928' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:41:55.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:55 smithi164 ceph-mon[49318]: pgmap v403: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:55.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:55 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/1220507928' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:41:57.402 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-09-10T03:41:57.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:57 smithi164 ceph-mon[49318]: pgmap v404: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:57.745 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:57 smithi130 ceph-mon[63591]: pgmap v404: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:41:58.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -e sha1=3a5e3f16720f432aea965c40a8f6eb480095ec6c -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-10T03:41:58.609 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:58 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/281088277' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:41:58.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:58 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/281088277' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:41:59.595 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:41:59 smithi130 ceph-mon[63591]: pgmap v405: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:41:59.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:41:59 smithi164 ceph-mon[49318]: pgmap v405: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:42:00.140 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.3.0-4885-g3a5e3f16 (3a5e3f16720f432aea965c40a8f6eb480095ec6c) squid (dev)" 2024-09-10T03:42:00.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:00 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:42:00.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:00 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/811087882' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:42:00.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:00 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-10T03:42:00.763 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:00 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/811087882' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-10T03:42:00.829 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-10T03:42:00.830 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-10T03:42:00.830 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- ceph orch ls -f json 2024-09-10T03:42:01.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:01 smithi164 ceph-mon[49318]: pgmap v406: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:42:01.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:01 smithi130 ceph-mon[63591]: pgmap v406: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:42:02.866 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-09-10T03:42:02.866 INFO:teuthology.orchestra.run.smithi130.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-10T03:19:41.509427Z", "last_refresh": "2024-09-10T03:40:02.342271Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-10T03:19:35.290015Z", "last_refresh": "2024-09-10T03:39:36.335140Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-10T03:19:38.252027Z", "last_refresh": "2024-09-10T03:40:02.342410Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-10T03:26:13.396434Z", "last_refresh": "2024-09-10T03:39:36.335697Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-10T03:19:33.242280Z", "last_refresh": "2024-09-10T03:39:36.335249Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi130:172.21.15.130=smithi130", "smithi164:172.21.15.164=smithi164"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-10T03:20:24.405687Z", "last_refresh": "2024-09-10T03:39:36.335317Z", "running": 2, "size": 2}}, {"events": ["2024-09-10T03:41:11.464139Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-10T03:26:34.217216Z", "last_refresh": "2024-09-10T03:39:36.335769Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-10T03:19:40.299038Z", "last_refresh": "2024-09-10T03:39:36.335382Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca", "last_refresh": "2024-09-10T03:39:36.335445Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-10T03:19:37.032460Z", "last_refresh": "2024-09-10T03:40:02.342544Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-10T03:42:03.471 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:03 smithi130 ceph-mon[63591]: pgmap v407: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:42:03.495 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-10T03:42:03.495 DEBUG:teuthology.parallel:result is None 2024-09-10T03:42:03.495 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-10T03:42:03.505 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi130.front.sepia.ceph.com 2024-09-10T03:42:03.505 DEBUG:teuthology.orchestra.run.smithi130:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-10T03:42:03.531 INFO:teuthology.orchestra.run.smithi130.stderr:+ umount /mnt/foo 2024-09-10T03:42:03.551 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-10T03:42:03.561 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi130.front.sepia.ceph.com 2024-09-10T03:42:03.561 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-10T03:42:03.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:03 smithi164 ceph-mon[49318]: pgmap v407: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:42:04.513 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:04 smithi130 ceph-mon[63591]: from='client.34630 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-10T03:42:04.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:04 smithi164 ceph-mon[49318]: from='client.34630 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-10T03:42:05.580 INFO:teuthology.orchestra.run.smithi130.stdout: "foo" 2024-09-10T03:42:05.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:05 smithi164 ceph-mon[49318]: pgmap v408: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:42:05.762 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:05 smithi130 ceph-mon[63591]: pgmap v408: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:42:06.233 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-10T03:42:07.483 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:07 smithi130 ceph-mon[63591]: from='client.34634 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:42:07.483 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:07 smithi130 ceph-mon[63591]: pgmap v409: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:42:07.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:07 smithi164 ceph-mon[49318]: from='client.34634 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:42:07.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:07 smithi164 ceph-mon[49318]: pgmap v409: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:42:08.283 INFO:teuthology.orchestra.run.smithi130.stdout:[ 2024-09-10T03:42:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-09-10T03:42:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: "access_type": "RW", 2024-09-10T03:42:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: "clients": [], 2024-09-10T03:42:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: "cluster_id": "foo", 2024-09-10T03:42:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: "export_id": 1, 2024-09-10T03:42:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: "fsal": { 2024-09-10T03:42:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: "fs_name": "foofs", 2024-09-10T03:42:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: "name": "CEPH", 2024-09-10T03:42:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: "user_id": "nfs.foo.1" 2024-09-10T03:42:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-09-10T03:42:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: "path": "/", 2024-09-10T03:42:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: "protocols": [ 2024-09-10T03:42:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: 4 2024-09-10T03:42:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-09-10T03:42:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: "pseudo": "/fake", 2024-09-10T03:42:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: "security_label": true, 2024-09-10T03:42:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: "squash": "none", 2024-09-10T03:42:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: "transports": [ 2024-09-10T03:42:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: "TCP" 2024-09-10T03:42:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-09-10T03:42:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-09-10T03:42:08.285 INFO:teuthology.orchestra.run.smithi130.stdout:] 2024-09-10T03:42:08.648 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:08 smithi130 ceph-mon[63591]: pgmap v410: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:42:08.648 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:08 smithi130 ceph-mon[63591]: from='client.34638 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:42:08.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:08 smithi164 ceph-mon[49318]: pgmap v410: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-10T03:42:08.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:08 smithi164 ceph-mon[49318]: from='client.34638 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-10T03:42:08.963 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-09-10T03:42:10.725 INFO:teuthology.orchestra.run.smithi130.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-10T03:42:10.725 INFO:teuthology.orchestra.run.smithi130.stdout:foo grace 2024-09-10T03:42:10.725 INFO:teuthology.orchestra.run.smithi130.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-10T03:42:10.725 INFO:teuthology.orchestra.run.smithi130.stdout:foo conf-nfs.foo 2024-09-10T03:42:10.725 INFO:teuthology.orchestra.run.smithi130.stdout:foo export-1 2024-09-10T03:42:11.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:10 smithi164 ceph-mon[49318]: pgmap v411: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:42:11.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:10 smithi164 ceph-mon[49318]: mgrmap e44: smithi130.axcmel(active, since 10m), standbys: smithi164.bkfwap 2024-09-10T03:42:11.262 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:10 smithi130 ceph-mon[63591]: pgmap v411: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:42:11.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:10 smithi130 ceph-mon[63591]: mgrmap e44: smithi130.axcmel(active, since 10m), standbys: smithi164.bkfwap 2024-09-10T03:42:11.377 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-10T03:42:11.388 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi130.front.sepia.ceph.com 2024-09-10T03:42:11.388 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5829498e-6f23-11ef-bcea-c7b262605968 -- bash -c 'set -ex 2024-09-10T03:42:11.388 DEBUG:teuthology.orchestra.run.smithi130:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-10T03:42:11.388 DEBUG:teuthology.orchestra.run.smithi130:> ' 2024-09-10T03:42:11.957 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:11 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:42:12.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:11 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-10T03:42:12.946 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:12 smithi130 ceph-mon[63591]: pgmap v412: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:42:12.946 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:12 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:42:12.946 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:12 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:42:12.946 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:12 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:42:12.946 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:12 smithi130 ceph-mon[63591]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:42:13.155 INFO:teuthology.orchestra.run.smithi130.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-10T03:42:13.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:12 smithi164 ceph-mon[49318]: pgmap v412: 97 pgs: 97 active+clean; 231 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-10T03:42:13.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:12 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-10T03:42:13.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:12 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-10T03:42:13.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:12 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:42:13.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:12 smithi164 ceph-mon[49318]: from='mgr.34100 172.21.15.130:0/4094626889' entity='mgr.smithi130.axcmel' 2024-09-10T03:42:13.408 INFO:teuthology.orchestra.run.smithi130.stderr:+ [[ 7 -gt 2 ]] 2024-09-10T03:42:14.012 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:13 smithi130 ceph-mon[63591]: from='client.? 172.21.15.130:0/669932246' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-10T03:42:14.076 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-10T03:42:14.086 INFO:tasks.cephadm:Teardown begin 2024-09-10T03:42:14.086 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-10T03:42:14.115 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-10T03:42:14.149 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-10T03:42:14.149 DEBUG:teuthology.orchestra.run.smithi130:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-10T03:42:14.170 DEBUG:teuthology.orchestra.run.smithi164:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-10T03:42:14.206 INFO:tasks.cephadm:Stopping all daemons... 2024-09-10T03:42:14.206 INFO:tasks.cephadm.mon.smithi130:Stopping mon.smithi130... 2024-09-10T03:42:14.206 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-5829498e-6f23-11ef-bcea-c7b262605968@mon.smithi130 2024-09-10T03:42:14.207 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:13 smithi164 ceph-mon[49318]: from='client.? 172.21.15.130:0/669932246' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-10T03:42:14.512 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:14 smithi130 systemd[1]: Stopping Ceph mon.smithi130 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:42:14.964 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:14 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130[63587]: 2024-09-10T03:42:14.510+0000 7ff033b10640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi130 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:42:14.965 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:14 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130[63587]: 2024-09-10T03:42:14.511+0000 7ff033b10640 -1 mon.smithi130@0(leader) e3 *** Got Signal Terminated *** 2024-09-10T03:42:14.965 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:14 smithi130 podman[92903]: 2024-09-10 03:42:14.695666461 +0000 UTC m=+0.296185386 container died 22e5cf56527a49da6464cb3ff752d169e9c5add018f7978f02b85e633b6f35ce (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, ceph=True, GIT_BRANCH=HEAD) 2024-09-10T03:42:15.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:15 smithi130 podman[92903]: 2024-09-10 03:42:15.070377916 +0000 UTC m=+0.670896829 container cleanup 22e5cf56527a49da6464cb3ff752d169e9c5add018f7978f02b85e633b6f35ce (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-10T03:42:15.263 INFO:journalctl@ceph.mon.smithi130.smithi130.stdout:Sep 10 03:42:15 smithi130 bash[92903]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi130 2024-09-10T03:42:15.501 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@mon.smithi130.service' 2024-09-10T03:42:15.544 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-10T03:42:15.544 INFO:tasks.cephadm.mon.smithi130:Stopped mon.smithi130 2024-09-10T03:42:15.544 INFO:tasks.cephadm.mon.smithi164:Stopping mon.smithi164... 2024-09-10T03:42:15.544 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ceph-5829498e-6f23-11ef-bcea-c7b262605968@mon.smithi164 2024-09-10T03:42:15.940 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:15 smithi164 systemd[1]: Stopping Ceph mon.smithi164 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:42:16.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:15 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi164[49314]: 2024-09-10T03:42:15.938+0000 7fab640f1640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi164 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:42:16.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:15 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi164[49314]: 2024-09-10T03:42:15.938+0000 7fab640f1640 -1 mon.smithi164@1(peon) e3 *** Got Signal Terminated *** 2024-09-10T03:42:16.208 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:16 smithi164 podman[68366]: 2024-09-10 03:42:16.030841911 +0000 UTC m=+0.287068041 container died 7d80bd0d15b99065d568954cf658e5c67afe9ad8bd6aa29c7a958be0168d0b34 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi164, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-10T03:42:16.707 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:16 smithi164 podman[68366]: 2024-09-10 03:42:16.522459854 +0000 UTC m=+0.778686026 container cleanup 7d80bd0d15b99065d568954cf658e5c67afe9ad8bd6aa29c7a958be0168d0b34 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi164, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, CEPH_POINT_RELEASE=) 2024-09-10T03:42:16.708 INFO:journalctl@ceph.mon.smithi164.smithi164.stdout:Sep 10 03:42:16 smithi164 bash[68366]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-mon-smithi164 2024-09-10T03:42:16.874 DEBUG:teuthology.orchestra.run.smithi164:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@mon.smithi164.service' 2024-09-10T03:42:16.921 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-10T03:42:16.921 INFO:tasks.cephadm.mon.smithi164:Stopped mon.smithi164 2024-09-10T03:42:16.921 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-10T03:42:16.921 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.0 2024-09-10T03:42:17.234 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:42:16 smithi130 systemd[1]: Stopping Ceph osd.0 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:42:17.513 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:42:17 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0[68339]: 2024-09-10T03:42:17.232+0000 7f77b04b1640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:42:17.513 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:42:17 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0[68339]: 2024-09-10T03:42:17.232+0000 7f77b04b1640 -1 osd.0 161 *** Got signal Terminated *** 2024-09-10T03:42:17.513 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:42:17 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0[68339]: 2024-09-10T03:42:17.232+0000 7f77b04b1640 -1 osd.0 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-10T03:42:22.945 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:42:22 smithi130 podman[93034]: 2024-09-10 03:42:22.642151144 +0000 UTC m=+5.535166938 container died de3cbff0098b445a5127e8eac4942621716538b9faecfde12405069e5fdad44e (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:42:23.262 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:42:23 smithi130 podman[93034]: 2024-09-10 03:42:23.040738885 +0000 UTC m=+5.933754685 container cleanup de3cbff0098b445a5127e8eac4942621716538b9faecfde12405069e5fdad44e (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-10T03:42:23.263 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:42:23 smithi130 bash[93034]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0 2024-09-10T03:42:23.658 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:42:23 smithi130 podman[93066]: 2024-09-10 03:42:23.292138321 +0000 UTC m=+0.645560652 container remove de3cbff0098b445a5127e8eac4942621716538b9faecfde12405069e5fdad44e (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-10T03:42:23.913 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:42:23 smithi130 podman[93154]: 2024-09-10 03:42:23.560797505 +0000 UTC m=+0.016301704 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:42:23.913 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:42:23 smithi130 podman[93154]: 2024-09-10 03:42:23.684999624 +0000 UTC m=+0.140503864 container create afff278b49e7426cea8d989a76e2e8355875a3ddb8bff9fc299f94090a8688af (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-09-10T03:42:24.263 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:42:23 smithi130 podman[93154]: 2024-09-10 03:42:23.912245711 +0000 UTC m=+0.367749917 container init afff278b49e7426cea8d989a76e2e8355875a3ddb8bff9fc299f94090a8688af (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1) 2024-09-10T03:42:24.263 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:42:23 smithi130 podman[93154]: 2024-09-10 03:42:23.91570077 +0000 UTC m=+0.371205009 container start afff278b49e7426cea8d989a76e2e8355875a3ddb8bff9fc299f94090a8688af (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-10T03:42:24.263 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:42:23 smithi130 podman[93154]: 2024-09-10 03:42:23.987228936 +0000 UTC m=+0.442733132 container attach afff278b49e7426cea8d989a76e2e8355875a3ddb8bff9fc299f94090a8688af (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-10T03:42:24.263 INFO:journalctl@ceph.osd.0.smithi130.stdout:Sep 10 03:42:24 smithi130 podman[93154]: 2024-09-10 03:42:24.105485853 +0000 UTC m=+0.560990050 container died afff278b49e7426cea8d989a76e2e8355875a3ddb8bff9fc299f94090a8688af (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-10T03:42:24.761 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.0.service' 2024-09-10T03:42:24.802 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-10T03:42:24.802 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-10T03:42:24.802 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-10T03:42:24.803 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.1 2024-09-10T03:42:25.013 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:42:24 smithi130 systemd[1]: Stopping Ceph osd.1 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:42:25.513 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:42:25 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1[70820]: 2024-09-10T03:42:25.142+0000 7fc703037640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:42:25.513 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:42:25 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1[70820]: 2024-09-10T03:42:25.142+0000 7fc703037640 -1 osd.1 161 *** Got signal Terminated *** 2024-09-10T03:42:25.513 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:42:25 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1[70820]: 2024-09-10T03:42:25.142+0000 7fc703037640 -1 osd.1 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-10T03:42:30.795 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:42:30 smithi130 podman[93255]: 2024-09-10 03:42:30.520191595 +0000 UTC m=+5.489642500 container died f0722a6d1c90d60fb6eca54e1f5f30f2960475164a303b69600d5492867452d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True) 2024-09-10T03:42:31.178 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:42:30 smithi130 podman[93255]: 2024-09-10 03:42:30.87654958 +0000 UTC m=+5.846000484 container cleanup f0722a6d1c90d60fb6eca54e1f5f30f2960475164a303b69600d5492867452d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-10T03:42:31.178 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:42:30 smithi130 bash[93255]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1 2024-09-10T03:42:31.493 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:42:31 smithi130 podman[93273]: 2024-09-10 03:42:31.17778867 +0000 UTC m=+0.653579137 container remove f0722a6d1c90d60fb6eca54e1f5f30f2960475164a303b69600d5492867452d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-10T03:42:31.763 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:42:31 smithi130 podman[93363]: 2024-09-10 03:42:31.396185178 +0000 UTC m=+0.015949899 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:42:31.763 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:42:31 smithi130 podman[93363]: 2024-09-10 03:42:31.495187125 +0000 UTC m=+0.114951839 container create d5dd3eba2f6040aafe1b9e1df0875c6150fb82aea5ee087094d63256d73fdb5e (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-10T03:42:31.763 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:42:31 smithi130 podman[93363]: 2024-09-10 03:42:31.714178292 +0000 UTC m=+0.333943003 container init d5dd3eba2f6040aafe1b9e1df0875c6150fb82aea5ee087094d63256d73fdb5e (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-10T03:42:31.763 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:42:31 smithi130 podman[93363]: 2024-09-10 03:42:31.717452611 +0000 UTC m=+0.337217335 container start d5dd3eba2f6040aafe1b9e1df0875c6150fb82aea5ee087094d63256d73fdb5e (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-10T03:42:32.210 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:42:31 smithi130 podman[93363]: 2024-09-10 03:42:31.822600229 +0000 UTC m=+0.442364945 container attach d5dd3eba2f6040aafe1b9e1df0875c6150fb82aea5ee087094d63256d73fdb5e (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-10T03:42:32.210 INFO:journalctl@ceph.osd.1.smithi130.stdout:Sep 10 03:42:31 smithi130 podman[93363]: 2024-09-10 03:42:31.910462967 +0000 UTC m=+0.530227681 container died d5dd3eba2f6040aafe1b9e1df0875c6150fb82aea5ee087094d63256d73fdb5e (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-10T03:42:32.525 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.1.service' 2024-09-10T03:42:32.565 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-10T03:42:32.566 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-10T03:42:32.566 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-10T03:42:32.566 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.2 2024-09-10T03:42:32.762 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:42:32 smithi130 systemd[1]: Stopping Ceph osd.2 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:42:33.263 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:42:32 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2[73291]: 2024-09-10T03:42:32.887+0000 7fc947a89640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:42:33.263 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:42:32 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2[73291]: 2024-09-10T03:42:32.887+0000 7fc947a89640 -1 osd.2 161 *** Got signal Terminated *** 2024-09-10T03:42:33.263 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:42:32 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2[73291]: 2024-09-10T03:42:32.887+0000 7fc947a89640 -1 osd.2 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-10T03:42:38.564 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:42:38 smithi130 podman[93465]: 2024-09-10 03:42:38.273776432 +0000 UTC m=+5.492029265 container died a871d2ccf96efe15953b07f959ddd4e758b8a8c3c0db3dc5f696512da7851e38 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:42:38.830 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:42:38 smithi130 podman[93465]: 2024-09-10 03:42:38.661754736 +0000 UTC m=+5.880007726 container cleanup a871d2ccf96efe15953b07f959ddd4e758b8a8c3c0db3dc5f696512da7851e38 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:42:38.830 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:42:38 smithi130 bash[93465]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2 2024-09-10T03:42:39.228 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:42:38 smithi130 podman[93480]: 2024-09-10 03:42:38.829726671 +0000 UTC m=+0.559254592 container remove a871d2ccf96efe15953b07f959ddd4e758b8a8c3c0db3dc5f696512da7851e38 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1) 2024-09-10T03:42:39.484 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:42:39 smithi130 podman[93572]: 2024-09-10 03:42:39.130939322 +0000 UTC m=+0.016619057 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:42:39.485 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:42:39 smithi130 podman[93572]: 2024-09-10 03:42:39.238913254 +0000 UTC m=+0.124592990 container create e1ab082a5a4a04221bcb1a09511adb8fe52b4ee40bf5a3163d9d3af4eee9cd53 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-09-10T03:42:39.763 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:42:39 smithi130 podman[93572]: 2024-09-10 03:42:39.484067144 +0000 UTC m=+0.369746874 container init e1ab082a5a4a04221bcb1a09511adb8fe52b4ee40bf5a3163d9d3af4eee9cd53 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-10T03:42:39.763 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:42:39 smithi130 podman[93572]: 2024-09-10 03:42:39.487473993 +0000 UTC m=+0.373153720 container start e1ab082a5a4a04221bcb1a09511adb8fe52b4ee40bf5a3163d9d3af4eee9cd53 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-10T03:42:39.763 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:42:39 smithi130 podman[93572]: 2024-09-10 03:42:39.591565321 +0000 UTC m=+0.477245058 container attach e1ab082a5a4a04221bcb1a09511adb8fe52b4ee40bf5a3163d9d3af4eee9cd53 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-10T03:42:39.763 INFO:journalctl@ceph.osd.2.smithi130.stdout:Sep 10 03:42:39 smithi130 podman[93572]: 2024-09-10 03:42:39.680061947 +0000 UTC m=+0.565741674 container died e1ab082a5a4a04221bcb1a09511adb8fe52b4ee40bf5a3163d9d3af4eee9cd53 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-10T03:42:40.286 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.2.service' 2024-09-10T03:42:40.324 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-10T03:42:40.324 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-10T03:42:40.325 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-10T03:42:40.325 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.3 2024-09-10T03:42:40.513 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:42:40 smithi130 systemd[1]: Stopping Ceph osd.3 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:42:41.013 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:42:40 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3[75643]: 2024-09-10T03:42:40.647+0000 7fcf5442a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:42:41.013 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:42:40 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3[75643]: 2024-09-10T03:42:40.647+0000 7fcf5442a640 -1 osd.3 161 *** Got signal Terminated *** 2024-09-10T03:42:41.013 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:42:40 smithi130 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3[75643]: 2024-09-10T03:42:40.647+0000 7fcf5442a640 -1 osd.3 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-10T03:42:43.930 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:43 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:43.460+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:42:44.207 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:43 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:43.929+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.063508+0000 front 2024-09-10T03:42:19.063456+0000 (oldest deadline 2024-09-10T03:42:43.763315+0000) 2024-09-10T03:42:44.957 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:44 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:44.482+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:42:45.457 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:44 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:44.961+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.063508+0000 front 2024-09-10T03:42:19.063456+0000 (oldest deadline 2024-09-10T03:42:43.763315+0000) 2024-09-10T03:42:45.958 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:45 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:45.507+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:42:46.254 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:45 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:45.970+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.063508+0000 front 2024-09-10T03:42:19.063456+0000 (oldest deadline 2024-09-10T03:42:43.763315+0000) 2024-09-10T03:42:46.254 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:46 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:46.192+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:42:46.324 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:42:46 smithi130 podman[93676]: 2024-09-10 03:42:46.04608113 +0000 UTC m=+5.503422622 container died f2bc83e91fa2274e1f52f6e6de0e3731522db9d8237a3709c0e21014e3736cfe (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-10T03:42:46.515 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:46 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[53180]: 2024-09-10T03:42:46.253+0000 7f176ee9c640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:22.232558+0000 front 2024-09-10T03:42:22.232522+0000 (oldest deadline 2024-09-10T03:42:45.732386+0000) 2024-09-10T03:42:46.673 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:42:46 smithi130 podman[93676]: 2024-09-10 03:42:46.430500355 +0000 UTC m=+5.887841844 container cleanup f2bc83e91fa2274e1f52f6e6de0e3731522db9d8237a3709c0e21014e3736cfe (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-10T03:42:46.674 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:42:46 smithi130 bash[93676]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3 2024-09-10T03:42:46.957 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:46 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:46.514+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:42:46.996 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:42:46 smithi130 podman[93711]: 2024-09-10 03:42:46.673344469 +0000 UTC m=+0.623931879 container remove f2bc83e91fa2274e1f52f6e6de0e3731522db9d8237a3709c0e21014e3736cfe (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-10T03:42:47.221 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:46 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:46.964+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.063508+0000 front 2024-09-10T03:42:19.063456+0000 (oldest deadline 2024-09-10T03:42:43.763315+0000) 2024-09-10T03:42:47.263 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:42:46 smithi130 podman[93802]: 2024-09-10 03:42:46.899235976 +0000 UTC m=+0.016017110 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:42:47.263 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:42:47 smithi130 podman[93802]: 2024-09-10 03:42:47.007816789 +0000 UTC m=+0.124597916 container create 7d50cda495ff1be1b470ca5b3dd0badeae604b74cfe2d79bddaa269f82ee934f (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-10T03:42:47.263 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:42:47 smithi130 podman[93802]: 2024-09-10 03:42:47.226753078 +0000 UTC m=+0.343534202 container init 7d50cda495ff1be1b470ca5b3dd0badeae604b74cfe2d79bddaa269f82ee934f (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, CEPH_POINT_RELEASE=) 2024-09-10T03:42:47.263 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:42:47 smithi130 podman[93802]: 2024-09-10 03:42:47.230192376 +0000 UTC m=+0.346973501 container start 7d50cda495ff1be1b470ca5b3dd0badeae604b74cfe2d79bddaa269f82ee934f (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-10T03:42:47.478 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:47 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:47.241+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:42:47.479 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:47 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[53180]: 2024-09-10T03:42:47.220+0000 7f176ee9c640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:22.232558+0000 front 2024-09-10T03:42:22.232522+0000 (oldest deadline 2024-09-10T03:42:45.732386+0000) 2024-09-10T03:42:47.755 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:42:47 smithi130 podman[93802]: 2024-09-10 03:42:47.301828453 +0000 UTC m=+0.418609595 container attach 7d50cda495ff1be1b470ca5b3dd0badeae604b74cfe2d79bddaa269f82ee934f (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-10T03:42:47.755 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:42:47 smithi130 podman[93802]: 2024-09-10 03:42:47.41533666 +0000 UTC m=+0.532117802 container died 7d50cda495ff1be1b470ca5b3dd0badeae604b74cfe2d79bddaa269f82ee934f (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-10T03:42:47.919 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:47 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:47.477+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:42:48.007 INFO:journalctl@ceph.osd.3.smithi130.stdout:Sep 10 03:42:48 smithi130 podman[93825]: 2024-09-10 03:42:48.004704935 +0000 UTC m=+0.583440624 container remove 7d50cda495ff1be1b470ca5b3dd0badeae604b74cfe2d79bddaa269f82ee934f (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-10T03:42:48.028 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.3.service' 2024-09-10T03:42:48.066 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-10T03:42:48.066 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-10T03:42:48.066 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-10T03:42:48.066 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.4 2024-09-10T03:42:48.197 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:48 smithi164 systemd[1]: Stopping Ceph osd.4 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:42:48.197 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:47 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:47.918+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.063508+0000 front 2024-09-10T03:42:19.063456+0000 (oldest deadline 2024-09-10T03:42:43.763315+0000) 2024-09-10T03:42:48.450 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:48 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:48.195+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:42:48.451 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:48 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[53180]: 2024-09-10T03:42:48.259+0000 7f176ee9c640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:22.232558+0000 front 2024-09-10T03:42:22.232522+0000 (oldest deadline 2024-09-10T03:42:45.732386+0000) 2024-09-10T03:42:48.451 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:48 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[53180]: 2024-09-10T03:42:48.382+0000 7f177688f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:42:48.451 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:48 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[53180]: 2024-09-10T03:42:48.382+0000 7f177688f640 -1 osd.4 161 *** Got signal Terminated *** 2024-09-10T03:42:48.451 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:48 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[53180]: 2024-09-10T03:42:48.382+0000 7f177688f640 -1 osd.4 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-10T03:42:48.707 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:48 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:48.449+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:42:49.207 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:48 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:48.915+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.063508+0000 front 2024-09-10T03:42:19.063456+0000 (oldest deadline 2024-09-10T03:42:43.763315+0000) 2024-09-10T03:42:49.708 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:49 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[53180]: 2024-09-10T03:42:49.260+0000 7f176ee9c640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:22.232558+0000 front 2024-09-10T03:42:22.232522+0000 (oldest deadline 2024-09-10T03:42:45.732386+0000) 2024-09-10T03:42:49.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:49 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:49.422+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:42:49.708 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:49 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:49.208+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:42:50.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:50 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:50.185+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:42:50.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:49 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:49.938+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.063508+0000 front 2024-09-10T03:42:19.063456+0000 (oldest deadline 2024-09-10T03:42:43.763315+0000) 2024-09-10T03:42:50.707 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:50 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[53180]: 2024-09-10T03:42:50.263+0000 7f176ee9c640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:22.232558+0000 front 2024-09-10T03:42:22.232522+0000 (oldest deadline 2024-09-10T03:42:45.732386+0000) 2024-09-10T03:42:50.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:50 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:50.432+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:42:51.289 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:50 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:50.959+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.063508+0000 front 2024-09-10T03:42:19.063456+0000 (oldest deadline 2024-09-10T03:42:43.763315+0000) 2024-09-10T03:42:51.290 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:50 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:50.959+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:29.664376+0000 front 2024-09-10T03:42:29.664396+0000 (oldest deadline 2024-09-10T03:42:50.764231+0000) 2024-09-10T03:42:51.290 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:51 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:51.160+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:42:51.290 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:51 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:51.160+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:25.871085+0000 front 2024-09-10T03:42:25.871122+0000 (oldest deadline 2024-09-10T03:42:50.570917+0000) 2024-09-10T03:42:51.707 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:51 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[53180]: 2024-09-10T03:42:51.288+0000 7f176ee9c640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:22.232558+0000 front 2024-09-10T03:42:22.232522+0000 (oldest deadline 2024-09-10T03:42:45.732386+0000) 2024-09-10T03:42:51.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:51 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:51.466+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:42:52.207 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:51 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:51.945+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.063508+0000 front 2024-09-10T03:42:19.063456+0000 (oldest deadline 2024-09-10T03:42:43.763315+0000) 2024-09-10T03:42:52.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:51 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:51.945+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:29.664376+0000 front 2024-09-10T03:42:29.664396+0000 (oldest deadline 2024-09-10T03:42:50.764231+0000) 2024-09-10T03:42:52.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:52 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:52.122+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:42:52.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:52 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:52.122+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:25.871085+0000 front 2024-09-10T03:42:25.871122+0000 (oldest deadline 2024-09-10T03:42:50.570917+0000) 2024-09-10T03:42:52.707 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:52 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[53180]: 2024-09-10T03:42:52.320+0000 7f176ee9c640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:22.232558+0000 front 2024-09-10T03:42:22.232522+0000 (oldest deadline 2024-09-10T03:42:45.732386+0000) 2024-09-10T03:42:52.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:52 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:52.450+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:42:53.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:52 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:52.920+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.063508+0000 front 2024-09-10T03:42:19.063456+0000 (oldest deadline 2024-09-10T03:42:43.763315+0000) 2024-09-10T03:42:53.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:52 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:52.920+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:29.664376+0000 front 2024-09-10T03:42:29.664396+0000 (oldest deadline 2024-09-10T03:42:50.764231+0000) 2024-09-10T03:42:53.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:53 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:53.146+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:42:53.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:53 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:53.146+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:25.871085+0000 front 2024-09-10T03:42:25.871122+0000 (oldest deadline 2024-09-10T03:42:50.570917+0000) 2024-09-10T03:42:53.707 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:53 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4[53180]: 2024-09-10T03:42:53.312+0000 7f176ee9c640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:22.232558+0000 front 2024-09-10T03:42:22.232522+0000 (oldest deadline 2024-09-10T03:42:45.732386+0000) 2024-09-10T03:42:53.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:53 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:53.409+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:42:54.208 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:53 smithi164 podman[68519]: 2024-09-10 03:42:53.95693607 +0000 UTC m=+5.696837174 container died 5e1e1b5e1266185da08cad1865b597257ead3d1849c141d24ef78718837d5c9f (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-10T03:42:54.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:54 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:54.100+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:42:54.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:54 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:54.100+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:25.871085+0000 front 2024-09-10T03:42:25.871122+0000 (oldest deadline 2024-09-10T03:42:50.570917+0000) 2024-09-10T03:42:54.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:53 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:53.913+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.063508+0000 front 2024-09-10T03:42:19.063456+0000 (oldest deadline 2024-09-10T03:42:43.763315+0000) 2024-09-10T03:42:54.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:53 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:53.913+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:29.664376+0000 front 2024-09-10T03:42:29.664396+0000 (oldest deadline 2024-09-10T03:42:50.764231+0000) 2024-09-10T03:42:54.706 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:54 smithi164 podman[68519]: 2024-09-10 03:42:54.407724146 +0000 UTC m=+6.147625249 container cleanup 5e1e1b5e1266185da08cad1865b597257ead3d1849c141d24ef78718837d5c9f (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-10T03:42:54.706 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:54 smithi164 bash[68519]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4 2024-09-10T03:42:54.706 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:54 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:54.381+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:42:54.707 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:54 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:54.382+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:42:54.958 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:54 smithi164 podman[68537]: 2024-09-10 03:42:54.705936408 +0000 UTC m=+0.750753026 container remove 5e1e1b5e1266185da08cad1865b597257ead3d1849c141d24ef78718837d5c9f (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-10T03:42:54.958 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:54 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:54.922+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.063508+0000 front 2024-09-10T03:42:19.063456+0000 (oldest deadline 2024-09-10T03:42:43.763315+0000) 2024-09-10T03:42:54.958 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:54 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:54.922+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:29.664376+0000 front 2024-09-10T03:42:29.664396+0000 (oldest deadline 2024-09-10T03:42:50.764231+0000) 2024-09-10T03:42:55.354 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:55 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:55.148+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:42:55.354 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:55 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:55.148+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:25.871085+0000 front 2024-09-10T03:42:25.871122+0000 (oldest deadline 2024-09-10T03:42:50.570917+0000) 2024-09-10T03:42:55.355 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:55 smithi164 podman[68623]: 2024-09-10 03:42:54.927131485 +0000 UTC m=+0.015981082 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:42:55.355 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:55 smithi164 podman[68623]: 2024-09-10 03:42:55.035317169 +0000 UTC m=+0.124166824 container create 20b93c07beb62fb953693bff76e97e3fe10d8fc8bc840784762277d064055e9e (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-10T03:42:55.355 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:55 smithi164 podman[68623]: 2024-09-10 03:42:55.271093117 +0000 UTC m=+0.359942710 container init 20b93c07beb62fb953693bff76e97e3fe10d8fc8bc840784762277d064055e9e (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-deactivate, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:42:55.355 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:55 smithi164 podman[68623]: 2024-09-10 03:42:55.274406055 +0000 UTC m=+0.363255651 container start 20b93c07beb62fb953693bff76e97e3fe10d8fc8bc840784762277d064055e9e (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1) 2024-09-10T03:42:55.355 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:55 smithi164 podman[68623]: 2024-09-10 03:42:55.354267155 +0000 UTC m=+0.443116761 container attach 20b93c07beb62fb953693bff76e97e3fe10d8fc8bc840784762277d064055e9e (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:42:55.708 INFO:journalctl@ceph.osd.4.smithi164.stdout:Sep 10 03:42:55 smithi164 podman[68623]: 2024-09-10 03:42:55.470464913 +0000 UTC m=+0.559314507 container died 20b93c07beb62fb953693bff76e97e3fe10d8fc8bc840784762277d064055e9e (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-10T03:42:55.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:55 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:55.370+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:42:55.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:55 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:55.370+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:42:56.108 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:55 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:55.916+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.063508+0000 front 2024-09-10T03:42:19.063456+0000 (oldest deadline 2024-09-10T03:42:43.763315+0000) 2024-09-10T03:42:56.108 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:55 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:55.916+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:29.664376+0000 front 2024-09-10T03:42:29.664396+0000 (oldest deadline 2024-09-10T03:42:50.764231+0000) 2024-09-10T03:42:56.134 DEBUG:teuthology.orchestra.run.smithi164:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.4.service' 2024-09-10T03:42:56.173 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-10T03:42:56.173 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-10T03:42:56.173 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-10T03:42:56.173 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.5 2024-09-10T03:42:56.367 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:56 smithi164 systemd[1]: Stopping Ceph osd.5 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:42:56.367 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:56 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:56.165+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:42:56.367 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:56 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:56.165+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:25.871085+0000 front 2024-09-10T03:42:25.871122+0000 (oldest deadline 2024-09-10T03:42:50.570917+0000) 2024-09-10T03:42:56.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:56 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:56.380+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:42:56.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:56 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:56.380+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:42:56.708 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:56 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:56.511+0000 7fc3a5a91640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:42:56.709 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:56 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:56.511+0000 7fc3a5a91640 -1 osd.5 161 *** Got signal Terminated *** 2024-09-10T03:42:56.709 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:56 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:56.511+0000 7fc3a5a91640 -1 osd.5 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-10T03:42:57.316 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:57 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:57.117+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:42:57.317 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:57 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:57.117+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:25.871085+0000 front 2024-09-10T03:42:25.871122+0000 (oldest deadline 2024-09-10T03:42:50.570917+0000) 2024-09-10T03:42:57.317 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:56 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:56.960+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.063508+0000 front 2024-09-10T03:42:19.063456+0000 (oldest deadline 2024-09-10T03:42:43.763315+0000) 2024-09-10T03:42:57.317 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:56 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:56.960+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:29.664376+0000 front 2024-09-10T03:42:29.664396+0000 (oldest deadline 2024-09-10T03:42:50.764231+0000) 2024-09-10T03:42:57.707 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:57 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:57.382+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:42:57.707 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:57 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:57.382+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:42:58.207 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:57 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:57.935+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.063508+0000 front 2024-09-10T03:42:19.063456+0000 (oldest deadline 2024-09-10T03:42:43.763315+0000) 2024-09-10T03:42:58.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:57 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:57.935+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:29.664376+0000 front 2024-09-10T03:42:29.664396+0000 (oldest deadline 2024-09-10T03:42:50.764231+0000) 2024-09-10T03:42:58.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:58 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:58.159+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:42:58.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:58 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:58.159+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:25.871085+0000 front 2024-09-10T03:42:25.871122+0000 (oldest deadline 2024-09-10T03:42:50.570917+0000) 2024-09-10T03:42:58.707 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:58 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:58.401+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:42:58.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:58 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:58.401+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:42:59.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:58 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:58.928+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.063508+0000 front 2024-09-10T03:42:19.063456+0000 (oldest deadline 2024-09-10T03:42:43.763315+0000) 2024-09-10T03:42:59.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:58 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:58.928+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:29.664376+0000 front 2024-09-10T03:42:29.664396+0000 (oldest deadline 2024-09-10T03:42:50.764231+0000) 2024-09-10T03:42:59.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:59 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:59.145+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:42:59.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:42:59 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:42:59.145+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:25.871085+0000 front 2024-09-10T03:42:25.871122+0000 (oldest deadline 2024-09-10T03:42:50.570917+0000) 2024-09-10T03:42:59.707 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:59 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:59.396+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:42:59.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:42:59 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:42:59.397+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:43:00.351 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:00 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:00.183+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:43:00.351 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:00 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:00.183+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:25.871085+0000 front 2024-09-10T03:42:25.871122+0000 (oldest deadline 2024-09-10T03:42:50.570917+0000) 2024-09-10T03:43:00.351 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:00 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:00.183+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:35.772279+0000 front 2024-09-10T03:42:35.772324+0000 (oldest deadline 2024-09-10T03:42:59.272115+0000) 2024-09-10T03:43:00.351 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:59 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:59.970+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.063508+0000 front 2024-09-10T03:42:19.063456+0000 (oldest deadline 2024-09-10T03:42:43.763315+0000) 2024-09-10T03:43:00.352 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:59 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:59.970+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:29.664376+0000 front 2024-09-10T03:42:29.664396+0000 (oldest deadline 2024-09-10T03:42:50.764231+0000) 2024-09-10T03:43:00.352 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:42:59 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:42:59.970+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:36.065153+0000 front 2024-09-10T03:42:36.065249+0000 (oldest deadline 2024-09-10T03:42:59.565091+0000) 2024-09-10T03:43:00.707 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:00 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:00.350+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:43:00.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:00 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:00.350+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:43:00.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:00 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:00.350+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:34.803302+0000 front 2024-09-10T03:42:34.803221+0000 (oldest deadline 2024-09-10T03:42:59.503115+0000) 2024-09-10T03:43:01.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:43:00 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:43:00.932+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.063508+0000 front 2024-09-10T03:42:19.063456+0000 (oldest deadline 2024-09-10T03:42:43.763315+0000) 2024-09-10T03:43:01.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:43:00 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:43:00.932+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:29.664376+0000 front 2024-09-10T03:42:29.664396+0000 (oldest deadline 2024-09-10T03:42:50.764231+0000) 2024-09-10T03:43:01.208 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:43:00 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5[55201]: 2024-09-10T03:43:00.932+0000 7fc3a20a6640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:36.065153+0000 front 2024-09-10T03:42:36.065249+0000 (oldest deadline 2024-09-10T03:42:59.565091+0000) 2024-09-10T03:43:01.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:01 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:01.135+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:43:01.209 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:01 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:01.135+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:25.871085+0000 front 2024-09-10T03:42:25.871122+0000 (oldest deadline 2024-09-10T03:42:50.570917+0000) 2024-09-10T03:43:01.209 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:01 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:01.135+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:35.772279+0000 front 2024-09-10T03:42:35.772324+0000 (oldest deadline 2024-09-10T03:42:59.272115+0000) 2024-09-10T03:43:01.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:01 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:01.322+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:43:01.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:01 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:01.322+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:43:01.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:01 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:01.322+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:34.803302+0000 front 2024-09-10T03:42:34.803221+0000 (oldest deadline 2024-09-10T03:42:59.503115+0000) 2024-09-10T03:43:02.287 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:43:02 smithi164 podman[68727]: 2024-09-10 03:43:02.023000831 +0000 UTC m=+5.611984424 container died 76ab1aa88b44a55155f9e824a08b98ce81e765f09872ea21b4ffdd8ea0e7e0b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_CLEAN=True) 2024-09-10T03:43:02.288 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:02 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:02.104+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:43:02.288 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:02 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:02.104+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:25.871085+0000 front 2024-09-10T03:42:25.871122+0000 (oldest deadline 2024-09-10T03:42:50.570917+0000) 2024-09-10T03:43:02.288 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:02 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:02.104+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:35.772279+0000 front 2024-09-10T03:42:35.772324+0000 (oldest deadline 2024-09-10T03:42:59.272115+0000) 2024-09-10T03:43:02.707 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:43:02 smithi164 podman[68727]: 2024-09-10 03:43:02.503184156 +0000 UTC m=+6.092167783 container cleanup 76ab1aa88b44a55155f9e824a08b98ce81e765f09872ea21b4ffdd8ea0e7e0b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-10T03:43:02.708 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:43:02 smithi164 bash[68727]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5 2024-09-10T03:43:02.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:02 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:02.331+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:43:02.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:02 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:02.331+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:43:02.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:02 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:02.331+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:34.803302+0000 front 2024-09-10T03:42:34.803221+0000 (oldest deadline 2024-09-10T03:42:59.503115+0000) 2024-09-10T03:43:03.057 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:43:02 smithi164 podman[68769]: 2024-09-10 03:43:02.762837888 +0000 UTC m=+0.738472468 container remove 76ab1aa88b44a55155f9e824a08b98ce81e765f09872ea21b4ffdd8ea0e7e0b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:43:03.319 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:43:03 smithi164 podman[68856]: 2024-09-10 03:43:03.000302455 +0000 UTC m=+0.015604460 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:43:03.319 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:43:03 smithi164 podman[68856]: 2024-09-10 03:43:03.205571032 +0000 UTC m=+0.220873045 container create 68c18ad777c6cac6281bbac259a64c2e7605a7c13aef95d8ec4d8f420d10e0dd (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux ) 2024-09-10T03:43:03.319 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:03 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:03.056+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:43:03.319 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:03 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:03.056+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:25.871085+0000 front 2024-09-10T03:42:25.871122+0000 (oldest deadline 2024-09-10T03:42:50.570917+0000) 2024-09-10T03:43:03.319 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:03 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:03.056+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:35.772279+0000 front 2024-09-10T03:42:35.772324+0000 (oldest deadline 2024-09-10T03:42:59.272115+0000) 2024-09-10T03:43:03.610 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:03 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:03.356+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:43:03.610 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:03 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:03.356+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:43:03.610 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:03 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:03.356+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:34.803302+0000 front 2024-09-10T03:42:34.803221+0000 (oldest deadline 2024-09-10T03:42:59.503115+0000) 2024-09-10T03:43:03.610 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:43:03 smithi164 podman[68856]: 2024-09-10 03:43:03.416149836 +0000 UTC m=+0.431451840 container init 68c18ad777c6cac6281bbac259a64c2e7605a7c13aef95d8ec4d8f420d10e0dd (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:43:03.610 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:43:03 smithi164 podman[68856]: 2024-09-10 03:43:03.419469244 +0000 UTC m=+0.434771249 container start 68c18ad777c6cac6281bbac259a64c2e7605a7c13aef95d8ec4d8f420d10e0dd (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True) 2024-09-10T03:43:03.610 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:43:03 smithi164 podman[68856]: 2024-09-10 03:43:03.49109304 +0000 UTC m=+0.506395045 container attach 68c18ad777c6cac6281bbac259a64c2e7605a7c13aef95d8ec4d8f420d10e0dd (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-10T03:43:03.957 INFO:journalctl@ceph.osd.5.smithi164.stdout:Sep 10 03:43:03 smithi164 podman[68856]: 2024-09-10 03:43:03.609922456 +0000 UTC m=+0.625224473 container died 68c18ad777c6cac6281bbac259a64c2e7605a7c13aef95d8ec4d8f420d10e0dd (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-5-deactivate, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-10T03:43:04.200 DEBUG:teuthology.orchestra.run.smithi164:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.5.service' 2024-09-10T03:43:04.240 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-10T03:43:04.241 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-10T03:43:04.241 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-10T03:43:04.241 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.6 2024-09-10T03:43:04.303 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:04 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:04.038+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:43:04.303 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:04 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:04.038+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:25.871085+0000 front 2024-09-10T03:42:25.871122+0000 (oldest deadline 2024-09-10T03:42:50.570917+0000) 2024-09-10T03:43:04.303 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:04 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:04.038+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:35.772279+0000 front 2024-09-10T03:42:35.772324+0000 (oldest deadline 2024-09-10T03:42:59.272115+0000) 2024-09-10T03:43:04.557 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:04 smithi164 systemd[1]: Stopping Ceph osd.6 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:43:04.557 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:04 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:04.556+0000 7f1a3288a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:43:04.558 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:04 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:04.322+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:43:04.558 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:04 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:04.322+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:43:04.558 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:04 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:04.322+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:34.803302+0000 front 2024-09-10T03:42:34.803221+0000 (oldest deadline 2024-09-10T03:42:59.503115+0000) 2024-09-10T03:43:04.957 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:04 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:04.556+0000 7f1a3288a640 -1 osd.6 161 *** Got signal Terminated *** 2024-09-10T03:43:04.957 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:04 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:04.556+0000 7f1a3288a640 -1 osd.6 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-10T03:43:05.342 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:05 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:05.045+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:43:05.342 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:05 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:05.045+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:25.871085+0000 front 2024-09-10T03:42:25.871122+0000 (oldest deadline 2024-09-10T03:42:50.570917+0000) 2024-09-10T03:43:05.342 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:05 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:05.045+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:35.772279+0000 front 2024-09-10T03:42:35.772324+0000 (oldest deadline 2024-09-10T03:42:59.272115+0000) 2024-09-10T03:43:05.707 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:05 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:05.341+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:43:05.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:05 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:05.341+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:43:05.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:05 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:05.341+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:34.803302+0000 front 2024-09-10T03:42:34.803221+0000 (oldest deadline 2024-09-10T03:42:59.503115+0000) 2024-09-10T03:43:06.363 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:06 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:06.018+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:43:06.363 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:06 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:06.018+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:25.871085+0000 front 2024-09-10T03:42:25.871122+0000 (oldest deadline 2024-09-10T03:42:50.570917+0000) 2024-09-10T03:43:06.363 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:06 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:06.018+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:35.772279+0000 front 2024-09-10T03:42:35.772324+0000 (oldest deadline 2024-09-10T03:42:59.272115+0000) 2024-09-10T03:43:06.707 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:06 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:06.362+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:43:06.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:06 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:06.362+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:43:06.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:06 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:06.362+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:34.803302+0000 front 2024-09-10T03:42:34.803221+0000 (oldest deadline 2024-09-10T03:42:59.503115+0000) 2024-09-10T03:43:07.207 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:06 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:06.985+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:43:07.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:06 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:06.985+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:25.871085+0000 front 2024-09-10T03:42:25.871122+0000 (oldest deadline 2024-09-10T03:42:50.570917+0000) 2024-09-10T03:43:07.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:06 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:06.985+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:35.772279+0000 front 2024-09-10T03:42:35.772324+0000 (oldest deadline 2024-09-10T03:42:59.272115+0000) 2024-09-10T03:43:07.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:07 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:07.410+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:43:07.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:07 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:07.410+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:43:07.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:07 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:07.410+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:34.803302+0000 front 2024-09-10T03:42:34.803221+0000 (oldest deadline 2024-09-10T03:42:59.503115+0000) 2024-09-10T03:43:08.384 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:07 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:07.957+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:43:08.384 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:07 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:07.957+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:25.871085+0000 front 2024-09-10T03:42:25.871122+0000 (oldest deadline 2024-09-10T03:42:50.570917+0000) 2024-09-10T03:43:08.384 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:07 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:07.957+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:35.772279+0000 front 2024-09-10T03:42:35.772324+0000 (oldest deadline 2024-09-10T03:42:59.272115+0000) 2024-09-10T03:43:08.707 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:08 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:08.383+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:43:08.707 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:08 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:08.383+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:43:08.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:08 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:08.383+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:34.803302+0000 front 2024-09-10T03:42:34.803221+0000 (oldest deadline 2024-09-10T03:42:59.503115+0000) 2024-09-10T03:43:08.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:08 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:08.383+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-09-10T03:42:44.804003+0000 front 2024-09-10T03:42:44.804024+0000 (oldest deadline 2024-09-10T03:43:08.303996+0000) 2024-09-10T03:43:09.207 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:08 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:08.928+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:20.570469+0000 front 2024-09-10T03:42:20.570539+0000 (oldest deadline 2024-09-10T03:42:45.870431+0000) 2024-09-10T03:43:09.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:08 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:08.928+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:25.871085+0000 front 2024-09-10T03:42:25.871122+0000 (oldest deadline 2024-09-10T03:42:50.570917+0000) 2024-09-10T03:43:09.208 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:08 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6[57220]: 2024-09-10T03:43:08.928+0000 7f1a2ee9f640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:35.772279+0000 front 2024-09-10T03:42:35.772324+0000 (oldest deadline 2024-09-10T03:42:59.272115+0000) 2024-09-10T03:43:09.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:09 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:09.406+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:43:09.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:09 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:09.406+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:43:09.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:09 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:09.406+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:34.803302+0000 front 2024-09-10T03:42:34.803221+0000 (oldest deadline 2024-09-10T03:42:59.503115+0000) 2024-09-10T03:43:09.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:09 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:09.406+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-09-10T03:42:44.804003+0000 front 2024-09-10T03:42:44.804024+0000 (oldest deadline 2024-09-10T03:43:08.303996+0000) 2024-09-10T03:43:10.348 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:10 smithi164 podman[68958]: 2024-09-10 03:43:10.088070135 +0000 UTC m=+5.624345049 container died ca90894c3878ac1695d1e9a5dbeb2a0b43bacc49c16a889a166873930799dec9 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-10T03:43:10.708 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:10 smithi164 podman[68958]: 2024-09-10 03:43:10.47229097 +0000 UTC m=+6.008565890 container cleanup ca90894c3878ac1695d1e9a5dbeb2a0b43bacc49c16a889a166873930799dec9 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:43:10.708 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:10 smithi164 bash[68958]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6 2024-09-10T03:43:10.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:10 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:10.390+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:43:10.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:10 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:10.390+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:43:10.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:10 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:10.390+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:34.803302+0000 front 2024-09-10T03:42:34.803221+0000 (oldest deadline 2024-09-10T03:42:59.503115+0000) 2024-09-10T03:43:10.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:10 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:10.390+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-09-10T03:42:44.804003+0000 front 2024-09-10T03:42:44.804024+0000 (oldest deadline 2024-09-10T03:43:08.303996+0000) 2024-09-10T03:43:11.041 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:10 smithi164 podman[68973]: 2024-09-10 03:43:10.723396425 +0000 UTC m=+0.637660542 container remove ca90894c3878ac1695d1e9a5dbeb2a0b43bacc49c16a889a166873930799dec9 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True) 2024-09-10T03:43:11.336 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:11 smithi164 podman[69065]: 2024-09-10 03:43:10.943372358 +0000 UTC m=+0.016061061 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:43:11.336 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:11 smithi164 podman[69065]: 2024-09-10 03:43:11.049595039 +0000 UTC m=+0.122283803 container create 30b9e078849a9743fd483630c06e4dba7329a7d408a2b107e139b6e15bf969a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-10T03:43:11.336 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:11 smithi164 podman[69065]: 2024-09-10 03:43:11.260365573 +0000 UTC m=+0.333054275 container init 30b9e078849a9743fd483630c06e4dba7329a7d408a2b107e139b6e15bf969a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-deactivate, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-10T03:43:11.336 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:11 smithi164 podman[69065]: 2024-09-10 03:43:11.263718573 +0000 UTC m=+0.336407280 container start 30b9e078849a9743fd483630c06e4dba7329a7d408a2b107e139b6e15bf969a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-10T03:43:11.336 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:11 smithi164 podman[69065]: 2024-09-10 03:43:11.335287869 +0000 UTC m=+0.407976589 container attach 30b9e078849a9743fd483630c06e4dba7329a7d408a2b107e139b6e15bf969a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-10T03:43:11.707 INFO:journalctl@ceph.osd.6.smithi164.stdout:Sep 10 03:43:11 smithi164 podman[69065]: 2024-09-10 03:43:11.455777023 +0000 UTC m=+0.528465725 container died 30b9e078849a9743fd483630c06e4dba7329a7d408a2b107e139b6e15bf969a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-09-10T03:43:11.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:11 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:11.369+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:43:11.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:11 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:11.369+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:43:11.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:11 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:11.369+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:34.803302+0000 front 2024-09-10T03:42:34.803221+0000 (oldest deadline 2024-09-10T03:42:59.503115+0000) 2024-09-10T03:43:11.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:11 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:11.369+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-09-10T03:42:44.804003+0000 front 2024-09-10T03:42:44.804024+0000 (oldest deadline 2024-09-10T03:43:08.303996+0000) 2024-09-10T03:43:12.036 DEBUG:teuthology.orchestra.run.smithi164:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.6.service' 2024-09-10T03:43:12.116 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-10T03:43:12.117 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-10T03:43:12.117 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-10T03:43:12.117 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.7 2024-09-10T03:43:12.420 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:12 smithi164 systemd[1]: Stopping Ceph osd.7 for 5829498e-6f23-11ef-bcea-c7b262605968... 2024-09-10T03:43:12.707 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:12 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:12.419+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:43:12.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:12 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:12.419+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:43:12.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:12 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:12.419+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:34.803302+0000 front 2024-09-10T03:42:34.803221+0000 (oldest deadline 2024-09-10T03:42:59.503115+0000) 2024-09-10T03:43:12.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:12 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:12.419+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-09-10T03:42:44.804003+0000 front 2024-09-10T03:42:44.804024+0000 (oldest deadline 2024-09-10T03:43:08.303996+0000) 2024-09-10T03:43:12.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:12 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:12.451+0000 7f73da3c6640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-10T03:43:12.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:12 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:12.451+0000 7f73da3c6640 -1 osd.7 161 *** Got signal Terminated *** 2024-09-10T03:43:12.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:12 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:12.451+0000 7f73da3c6640 -1 osd.7 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-10T03:43:13.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:13 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:13.449+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:43:13.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:13 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:13.449+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:43:13.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:13 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:13.449+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:34.803302+0000 front 2024-09-10T03:42:34.803221+0000 (oldest deadline 2024-09-10T03:42:59.503115+0000) 2024-09-10T03:43:13.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:13 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:13.449+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-09-10T03:42:44.804003+0000 front 2024-09-10T03:42:44.804024+0000 (oldest deadline 2024-09-10T03:43:08.303996+0000) 2024-09-10T03:43:14.785 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:14 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:14.476+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:43:14.785 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:14 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:14.476+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:43:14.785 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:14 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:14.476+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:34.803302+0000 front 2024-09-10T03:42:34.803221+0000 (oldest deadline 2024-09-10T03:42:59.503115+0000) 2024-09-10T03:43:14.785 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:14 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:14.476+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-09-10T03:42:44.804003+0000 front 2024-09-10T03:42:44.804024+0000 (oldest deadline 2024-09-10T03:43:08.303996+0000) 2024-09-10T03:43:14.785 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:14 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:14.476+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.164:6804 osd.4 since back 2024-09-10T03:42:53.004800+0000 front 2024-09-10T03:42:53.004922+0000 (oldest deadline 2024-09-10T03:43:14.104824+0000) 2024-09-10T03:43:15.957 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:15 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:15.515+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:43:15.958 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:15 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:15.515+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:43:15.958 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:15 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:15.515+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:34.803302+0000 front 2024-09-10T03:42:34.803221+0000 (oldest deadline 2024-09-10T03:42:59.503115+0000) 2024-09-10T03:43:15.958 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:15 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:15.515+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-09-10T03:42:44.804003+0000 front 2024-09-10T03:42:44.804024+0000 (oldest deadline 2024-09-10T03:43:08.303996+0000) 2024-09-10T03:43:15.958 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:15 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:15.515+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.164:6804 osd.4 since back 2024-09-10T03:42:53.004800+0000 front 2024-09-10T03:42:53.004922+0000 (oldest deadline 2024-09-10T03:43:14.104824+0000) 2024-09-10T03:43:16.957 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:16 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:16.516+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-09-10T03:42:19.601441+0000 front 2024-09-10T03:42:19.601365+0000 (oldest deadline 2024-09-10T03:42:43.101314+0000) 2024-09-10T03:43:16.958 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:16 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:16.516+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-09-10T03:42:28.402474+0000 front 2024-09-10T03:42:28.402527+0000 (oldest deadline 2024-09-10T03:42:53.702384+0000) 2024-09-10T03:43:16.958 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:16 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:16.516+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-09-10T03:42:34.803302+0000 front 2024-09-10T03:42:34.803221+0000 (oldest deadline 2024-09-10T03:42:59.503115+0000) 2024-09-10T03:43:16.958 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:16 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:16.516+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-09-10T03:42:44.804003+0000 front 2024-09-10T03:42:44.804024+0000 (oldest deadline 2024-09-10T03:43:08.303996+0000) 2024-09-10T03:43:16.958 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:16 smithi164 ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7[59254]: 2024-09-10T03:43:16.516+0000 7f73d29d3640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.164:6804 osd.4 since back 2024-09-10T03:42:53.004800+0000 front 2024-09-10T03:42:53.004922+0000 (oldest deadline 2024-09-10T03:43:14.104824+0000) 2024-09-10T03:43:18.382 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:17 smithi164 podman[69168]: 2024-09-10 03:43:17.983272681 +0000 UTC m=+5.642898747 container died 69f0162ef515ee3e19c9b2c551abab11be7ec9777541799041767e6e5f4a6d66 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1) 2024-09-10T03:43:18.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:18 smithi164 podman[69168]: 2024-09-10 03:43:18.382730701 +0000 UTC m=+6.042356766 container cleanup 69f0162ef515ee3e19c9b2c551abab11be7ec9777541799041767e6e5f4a6d66 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-10T03:43:18.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:18 smithi164 bash[69168]: ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7 2024-09-10T03:43:18.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:18 smithi164 podman[69196]: 2024-09-10 03:43:18.617272545 +0000 UTC m=+0.637363242 container remove 69f0162ef515ee3e19c9b2c551abab11be7ec9777541799041767e6e5f4a6d66 (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-10T03:43:19.208 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:18 smithi164 podman[69287]: 2024-09-10 03:43:18.857591302 +0000 UTC m=+0.016420905 image pull d6d839bc8a4bdc14017733b053ef2e9b386c4d0f48b6d189c0888843abe03578 quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca 2024-09-10T03:43:19.208 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:18 smithi164 podman[69287]: 2024-09-10 03:43:18.968088434 +0000 UTC m=+0.126918036 container create 5b5a6e20236c6c568a1a03faf289dad761e478940abd263f4341245f1e371ded (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-10T03:43:19.707 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:19 smithi164 podman[69287]: 2024-09-10 03:43:19.212097842 +0000 UTC m=+0.370927445 container init 5b5a6e20236c6c568a1a03faf289dad761e478940abd263f4341245f1e371ded (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1) 2024-09-10T03:43:19.707 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:19 smithi164 podman[69287]: 2024-09-10 03:43:19.215285723 +0000 UTC m=+0.374115326 container start 5b5a6e20236c6c568a1a03faf289dad761e478940abd263f4341245f1e371ded (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_BRANCH=HEAD) 2024-09-10T03:43:19.707 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:19 smithi164 podman[69287]: 2024-09-10 03:43:19.295528508 +0000 UTC m=+0.454358125 container attach 5b5a6e20236c6c568a1a03faf289dad761e478940abd263f4341245f1e371ded (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-10T03:43:19.708 INFO:journalctl@ceph.osd.7.smithi164.stdout:Sep 10 03:43:19 smithi164 podman[69287]: 2024-09-10 03:43:19.411167899 +0000 UTC m=+0.569997505 container died 5b5a6e20236c6c568a1a03faf289dad761e478940abd263f4341245f1e371ded (image=quay.ceph.io/ceph-ci/ceph@sha256:2db6858ffc169715474ea237dd8b2beae440a5514686b8202506e12cb7dc1fca, name=ceph-5829498e-6f23-11ef-bcea-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-09-1904-3a5e3f1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-10T03:43:20.021 DEBUG:teuthology.orchestra.run.smithi164:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5829498e-6f23-11ef-bcea-c7b262605968@osd.7.service' 2024-09-10T03:43:20.059 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-10T03:43:20.059 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-10T03:43:20.059 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5829498e-6f23-11ef-bcea-c7b262605968 --force --keep-logs 2024-09-10T03:46:15.678 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5829498e-6f23-11ef-bcea-c7b262605968 --force --keep-logs 2024-09-10T03:46:35.517 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-10T03:46:35.552 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-10T03:46:35.581 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-10T03:46:35.583 DEBUG:teuthology.misc:Transferring archived files from smithi130:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/crash to /home/teuthworker/archive/adking-2024-09-10_02:59:23-orch:cephadm-wip-adk-testing-2024-09-09-1904-distro-default-smithi/7898560/remote/smithi130/crash 2024-09-10T03:46:35.584 DEBUG:teuthology.orchestra.run.smithi130:> sudo tar c -f - -C /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/crash -- . 2024-09-10T03:46:35.634 INFO:teuthology.orchestra.run.smithi130.stderr:tar: /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/crash: Cannot open: No such file or directory 2024-09-10T03:46:35.634 INFO:teuthology.orchestra.run.smithi130.stderr:tar: Error is not recoverable: exiting now 2024-09-10T03:46:35.636 DEBUG:teuthology.misc:Transferring archived files from smithi164:/var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/crash to /home/teuthworker/archive/adking-2024-09-10_02:59:23-orch:cephadm-wip-adk-testing-2024-09-09-1904-distro-default-smithi/7898560/remote/smithi164/crash 2024-09-10T03:46:35.637 DEBUG:teuthology.orchestra.run.smithi164:> sudo tar c -f - -C /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/crash -- . 2024-09-10T03:46:35.674 INFO:teuthology.orchestra.run.smithi164.stderr:tar: /var/lib/ceph/5829498e-6f23-11ef-bcea-c7b262605968/crash: Cannot open: No such file or directory 2024-09-10T03:46:35.674 INFO:teuthology.orchestra.run.smithi164.stderr:tar: Error is not recoverable: exiting now 2024-09-10T03:46:35.676 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-10T03:46:35.676 DEBUG:teuthology.orchestra.run.smithi130:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-10T03:46:35.706 INFO:tasks.cephadm:Compressing logs... 2024-09-10T03:46:35.707 DEBUG:teuthology.orchestra.run.smithi130:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-10T03:46:35.749 DEBUG:teuthology.orchestra.run.smithi164:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-10T03:46:35.773 INFO:teuthology.orchestra.run.smithi130.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.0.log 2024-09-10T03:46:35.773 INFO:teuthology.orchestra.run.smithi130.stderr:: No such file or directory 2024-09-10T03:46:35.773 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-mon.smithi130.log 2024-09-10T03:46:35.779 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.1.log 2024-09-10T03:46:35.779 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.0.log: /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-mon.smithi130.log: gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-mgr.smithi130.axcmel.log 2024-09-10T03:46:35.779 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-volume.log 2024-09-10T03:46:35.779 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph.audit.log 2024-09-10T03:46:35.779 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-mgr.smithi130.axcmel.log: /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-mds.foofs.smithi130.ukcxnb.log 2024-09-10T03:46:35.780 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph.cephadm.log 2024-09-10T03:46:35.780 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-mds.foofs.smithi130.ukcxnb.log: gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.2.log 2024-09-10T03:46:35.780 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph.cephadm.log: 85.8% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph.cephadm.log.gz 2024-09-10T03:46:35.780 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph.log 2024-09-10T03:46:35.782 INFO:teuthology.orchestra.run.smithi164.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.7.log 2024-09-10T03:46:35.783 INFO:teuthology.orchestra.run.smithi164.stderr:: No such file or directory 2024-09-10T03:46:35.783 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-volume.log 2024-09-10T03:46:35.783 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.4.log 2024-09-10T03:46:35.783 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph.audit.log 2024-09-10T03:46:35.783 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-mds.foofs.smithi164.awiezo.log 2024-09-10T03:46:35.783 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph.cephadm.log 2024-09-10T03:46:35.783 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph.audit.log: /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-mds.foofs.smithi164.awiezo.log: gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.6.log 2024-09-10T03:46:35.783 INFO:teuthology.orchestra.run.smithi164.stderr: 88.2% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-mds.foofs.smithi164.awiezo.log.gz 2024-09-10T03:46:35.783 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.5.log 2024-09-10T03:46:35.783 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5/var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.6.log: --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph.log 2024-09-10T03:46:35.783 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.5.log: 85.7%gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-mon.smithi164.log 2024-09-10T03:46:35.783 INFO:teuthology.orchestra.run.smithi164.stderr: -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph.cephadm.log.gz 2024-09-10T03:46:35.784 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-mgr.smithi164.bkfwap.log 2024-09-10T03:46:35.784 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph.log: /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-mon.smithi164.log: 91.6% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph.audit.log.gz 2024-09-10T03:46:35.784 INFO:teuthology.orchestra.run.smithi164.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-10T03:46:35.784 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-mgr.smithi164.bkfwap.log: 87.1% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph.log.gz 2024-09-10T03:46:35.785 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.2.log: 91.5% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph.audit.log.gz 2024-09-10T03:46:35.785 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.3.log 2024-09-10T03:46:35.787 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph.log: 91.1% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-volume.log.gz 2024-09-10T03:46:35.788 INFO:teuthology.orchestra.run.smithi130.stderr: 87.1% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph.log.gz 2024-09-10T03:46:35.789 INFO:teuthology.orchestra.run.smithi164.stderr: 91.2% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-volume.log.gz 2024-09-10T03:46:35.796 INFO:teuthology.orchestra.run.smithi164.stderr:/var/log/ceph/cephadm.log: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-10T03:46:35.797 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-10T03:46:35.800 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.3.log: /var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-mds.foofs.smithi130.ukcxnb.log.gz 2024-09-10T03:46:35.832 INFO:teuthology.orchestra.run.smithi164.stderr: 89.8% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-mgr.smithi164.bkfwap.log.gz 2024-09-10T03:46:35.837 INFO:teuthology.orchestra.run.smithi130.stderr: 91.7% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-10T03:46:36.154 INFO:teuthology.orchestra.run.smithi130.stderr: 88.7% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-mgr.smithi130.axcmel.log.gz 2024-09-10T03:46:36.254 INFO:teuthology.orchestra.run.smithi164.stderr: 92.6% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-mon.smithi164.log.gz 2024-09-10T03:46:37.154 INFO:teuthology.orchestra.run.smithi130.stderr: 91.8% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-mon.smithi130.log.gz 2024-09-10T03:46:48.078 INFO:teuthology.orchestra.run.smithi130.stderr: 94.5% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.2.log.gz 2024-09-10T03:46:50.481 INFO:teuthology.orchestra.run.smithi164.stderr: 93.9% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.4.log.gz 2024-09-10T03:46:50.583 INFO:teuthology.orchestra.run.smithi164.stderr: 94.2% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.6.log.gz 2024-09-10T03:46:51.697 INFO:teuthology.orchestra.run.smithi130.stderr: 94.1% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.0.log.gz 2024-09-10T03:46:53.179 INFO:teuthology.orchestra.run.smithi130.stderr: 94.1% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.3.log.gz 2024-09-10T03:46:53.868 INFO:teuthology.orchestra.run.smithi130.stderr: 94.2% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.1.log.gz 2024-09-10T03:46:53.870 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-09-10T03:46:53.870 INFO:teuthology.orchestra.run.smithi130.stderr:real 0m18.107s 2024-09-10T03:46:53.870 INFO:teuthology.orchestra.run.smithi130.stderr:user 1m2.953s 2024-09-10T03:46:53.870 INFO:teuthology.orchestra.run.smithi130.stderr:sys 0m2.492s 2024-09-10T03:46:54.555 INFO:teuthology.orchestra.run.smithi164.stderr: 94.2% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.5.log.gz 2024-09-10T03:46:59.123 INFO:teuthology.orchestra.run.smithi164.stderr: 94.1% -- replaced with /var/log/ceph/5829498e-6f23-11ef-bcea-c7b262605968/ceph-osd.7.log.gz 2024-09-10T03:46:59.125 INFO:teuthology.orchestra.run.smithi164.stderr: 2024-09-10T03:46:59.125 INFO:teuthology.orchestra.run.smithi164.stderr:real 0m23.361s 2024-09-10T03:46:59.125 INFO:teuthology.orchestra.run.smithi164.stderr:user 1m9.200s 2024-09-10T03:46:59.125 INFO:teuthology.orchestra.run.smithi164.stderr:sys 0m2.887s 2024-09-10T03:46:59.125 INFO:tasks.cephadm:Archiving logs... 2024-09-10T03:46:59.126 DEBUG:teuthology.misc:Transferring archived files from smithi130:/var/log/ceph to /home/teuthworker/archive/adking-2024-09-10_02:59:23-orch:cephadm-wip-adk-testing-2024-09-09-1904-distro-default-smithi/7898560/remote/smithi130/log 2024-09-10T03:46:59.127 DEBUG:teuthology.orchestra.run.smithi130:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-10T03:47:07.110 DEBUG:teuthology.misc:Transferring archived files from smithi164:/var/log/ceph to /home/teuthworker/archive/adking-2024-09-10_02:59:23-orch:cephadm-wip-adk-testing-2024-09-09-1904-distro-default-smithi/7898560/remote/smithi164/log 2024-09-10T03:47:07.111 DEBUG:teuthology.orchestra.run.smithi164:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-10T03:47:16.717 INFO:tasks.cephadm:Removing cluster... 2024-09-10T03:47:16.717 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5829498e-6f23-11ef-bcea-c7b262605968 --force 2024-09-10T03:47:17.079 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5829498e-6f23-11ef-bcea-c7b262605968 --force 2024-09-10T03:47:17.459 INFO:tasks.cephadm:Removing cephadm ... 2024-09-10T03:47:17.460 DEBUG:teuthology.orchestra.run.smithi130:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-10T03:47:17.475 DEBUG:teuthology.orchestra.run.smithi164:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-10T03:47:17.490 INFO:tasks.cephadm:Teardown complete 2024-09-10T03:47:17.490 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-10T03:47:17.501 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-10T03:47:17.501 DEBUG:teuthology.orchestra.run.smithi130:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-10T03:47:17.518 DEBUG:teuthology.orchestra.run.smithi164:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-10T03:47:17.529 INFO:teuthology.orchestra.run.smithi130.stderr:bash: line 1: ntpq: command not found 2024-09-10T03:47:17.533 INFO:teuthology.orchestra.run.smithi130.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-10T03:47:17.533 INFO:teuthology.orchestra.run.smithi130.stdout:=============================================================================== 2024-09-10T03:47:17.533 INFO:teuthology.orchestra.run.smithi130.stdout:^* hv01.front.sepia.ceph.com 3 6 377 41 +1702ns[ +73us] +/- 54ms 2024-09-10T03:47:17.533 INFO:teuthology.orchestra.run.smithi130.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 40 -468us[ -468us] +/- 77ms 2024-09-10T03:47:17.533 INFO:teuthology.orchestra.run.smithi130.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 41 +741us[ +741us] +/- 62ms 2024-09-10T03:47:17.533 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-10T03:47:17.544 INFO:teuthology.orchestra.run.smithi164.stderr:bash: line 1: ntpq: command not found 2024-09-10T03:47:17.548 INFO:teuthology.orchestra.run.smithi164.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-10T03:47:17.548 INFO:teuthology.orchestra.run.smithi164.stdout:=============================================================================== 2024-09-10T03:47:17.548 INFO:teuthology.orchestra.run.smithi164.stdout:^* hv01.front.sepia.ceph.com 3 6 377 44 -31us[ +370ns] +/- 54ms 2024-09-10T03:47:17.548 INFO:teuthology.orchestra.run.smithi164.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 45 -492us[ -461us] +/- 77ms 2024-09-10T03:47:17.548 INFO:teuthology.orchestra.run.smithi164.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 43 +721us[ +721us] +/- 62ms 2024-09-10T03:47:17.548 INFO:teuthology.orchestra.run.smithi164.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-10T03:47:17.549 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-10T03:47:17.559 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-10T03:47:17.559 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-10T03:47:17.569 DEBUG:teuthology.orchestra.run.smithi130:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-10T03:47:17.650 DEBUG:teuthology.orchestra.run.smithi164:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-10T03:47:17.720 DEBUG:teuthology.orchestra.run.smithi130:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-10T03:47:17.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-10T03:47:17.749 DEBUG:teuthology.orchestra.run.smithi164:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-10T03:47:17.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-10T03:47:17.778 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-10T03:47:17.787 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-10T03:47:17.796 INFO:teuthology.task.internal:Duration was 1955.732187 seconds 2024-09-10T03:47:17.796 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-10T03:47:17.804 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-10T03:47:17.805 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-10T03:47:17.807 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-10T03:47:17.843 INFO:teuthology.orchestra.run.smithi130.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-10T03:47:17.857 INFO:teuthology.orchestra.run.smithi164.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-10T03:47:18.249 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-10T03:47:18.250 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi130.front.sepia.ceph.com 2024-09-10T03:47:18.250 DEBUG:teuthology.orchestra.run.smithi130:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-10T03:47:18.270 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi164.front.sepia.ceph.com 2024-09-10T03:47:18.270 DEBUG:teuthology.orchestra.run.smithi164:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-10T03:47:18.313 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-10T03:47:18.313 DEBUG:teuthology.orchestra.run.smithi130:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-10T03:47:18.315 DEBUG:teuthology.orchestra.run.smithi164:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-10T03:47:18.379 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-10T03:47:18.380 DEBUG:teuthology.orchestra.run.smithi130:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-10T03:47:18.382 DEBUG:teuthology.orchestra.run.smithi164:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-10T03:47:32.289 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-10T03:47:32.302 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-10T03:47:32.302 DEBUG:teuthology.orchestra.run.smithi130:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-10T03:47:32.327 DEBUG:teuthology.orchestra.run.smithi164:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-10T03:47:32.358 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-10T03:47:32.368 DEBUG:teuthology.orchestra.run.smithi130:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-10T03:47:32.371 DEBUG:teuthology.orchestra.run.smithi164:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-10T03:47:32.391 INFO:teuthology.orchestra.run.smithi130.stdout:kernel.core_pattern = core 2024-09-10T03:47:32.426 INFO:teuthology.orchestra.run.smithi164.stdout:kernel.core_pattern = core 2024-09-10T03:47:32.449 DEBUG:teuthology.orchestra.run.smithi130:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-10T03:47:32.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-10T03:47:32.475 DEBUG:teuthology.orchestra.run.smithi164:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-10T03:47:32.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-10T03:47:32.506 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-10T03:47:32.516 INFO:teuthology.task.internal:Transferring archived files... 2024-09-10T03:47:32.517 DEBUG:teuthology.misc:Transferring archived files from smithi130:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-09-10_02:59:23-orch:cephadm-wip-adk-testing-2024-09-09-1904-distro-default-smithi/7898560/remote/smithi130 2024-09-10T03:47:32.518 DEBUG:teuthology.orchestra.run.smithi130:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-10T03:47:32.579 DEBUG:teuthology.misc:Transferring archived files from smithi164:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-09-10_02:59:23-orch:cephadm-wip-adk-testing-2024-09-09-1904-distro-default-smithi/7898560/remote/smithi164 2024-09-10T03:47:32.580 DEBUG:teuthology.orchestra.run.smithi164:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-10T03:47:32.621 INFO:teuthology.task.internal:Removing archive directory... 2024-09-10T03:47:32.621 DEBUG:teuthology.orchestra.run.smithi130:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-10T03:47:32.624 DEBUG:teuthology.orchestra.run.smithi164:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-10T03:47:32.671 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-10T03:47:32.683 INFO:teuthology.task.internal:Not uploading archives. 2024-09-10T03:47:32.683 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-10T03:47:32.693 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-10T03:47:32.693 DEBUG:teuthology.orchestra.run.smithi130:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-10T03:47:32.695 DEBUG:teuthology.orchestra.run.smithi164:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-10T03:47:32.708 INFO:teuthology.orchestra.run.smithi130.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 10 03:47 /home/ubuntu/cephtest 2024-09-10T03:47:32.727 INFO:teuthology.orchestra.run.smithi164.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 10 03:47 /home/ubuntu/cephtest 2024-09-10T03:47:32.729 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-10T03:47:32.740 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-10T03:47:32.764 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1955.7321870326996 owner: scheduled_adking@teuthology success: true 2024-09-10T03:47:32.765 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-10T03:47:32.842 INFO:teuthology.run:pass