2024-05-03T08:43:42.351 INFO:root:teuthology version: 0.0.1.dev271+g1ae7ad8 2024-05-03T08:43:42.352 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_gabrioux@teuthology --description orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} --name gabrioux-2024-05-03_06:31:58-orch:cephadm-wip-guits-testing-2024-05-02-0628-distro-default-smithi --archive /home/teuthworker/archive/gabrioux-2024-05-03_06:31:58-orch:cephadm-wip-guits-testing-2024-05-02-0628-distro-default-smithi/7687215 --verbose -- /home/teuthworker/archive/gabrioux-2024-05-03_06:31:58-orch:cephadm-wip-guits-testing-2024-05-02-0628-distro-default-smithi/7687215/orig.config.yaml 2024-05-03T08:43:42.390 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-05-03T08:43:42.477 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/gabrioux-2024-05-03_06:31:58-orch:cephadm-wip-guits-testing-2024-05-02-0628-distro-default-smithi/7687215 branch: wip-guits-testing-2024-05-02-0628 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: gabrioux@ibm.com first_in_suite: false job_id: '7687215' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: gabrioux-2024-05-03_06:31:58-orch:cephadm-wip-guits-testing-2024-05-02-0628-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-guits-testing-2024-05-02-0628 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: f8e6d2d721bc96a93915917c59bc3d287d57263d ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: f8e6d2d721bc96a93915917c59bc3d287d57263d selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: wip-guits-testing-2024-05-02-0628 sha1: f8e6d2d721bc96a93915917c59bc3d287d57263d owner: scheduled_gabrioux@teuthology priority: 70 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 2184 sha1: f8e6d2d721bc96a93915917c59bc3d287d57263d sleep_before_teardown: 0 subset: 1/8 suite: orch:cephadm suite_branch: wip-guits-testing-2024-05-02-0628 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_f8e6d2d721bc96a93915917c59bc3d287d57263d/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: f8e6d2d721bc96a93915917c59bc3d287d57263d targets: smithi032.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEYrsD74lBa50O7lCv8p0GIMhm9Xi+BbpVGMVX9pOlxGLVWXvq1FiPFSf/r9uExnjfKzFofHp+R3y0s3PiB/8H0= smithi123.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIZAxocJHzqm+vJfluTD1dI9ijqhNKBoFjLanVcpN0pIto+9rKftW5yCQJ13k8hZ/j02C2n+cH+NEG0lbFvvfNE= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 1ae7ad82388e92a475afff437d49054826c019a1 timestamp: 2024-05-03_06:31:58 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: gabrioux verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-05-03T08:43:42.477 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_f8e6d2d721bc96a93915917c59bc3d287d57263d/qa; will attempt to use it 2024-05-03T08:43:42.478 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_f8e6d2d721bc96a93915917c59bc3d287d57263d/qa/tasks 2024-05-03T08:43:42.478 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-05-03T08:43:42.479 INFO:teuthology.task.internal:Checking packages... 2024-05-03T08:43:42.506 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'f8e6d2d721bc96a93915917c59bc3d287d57263d' 2024-05-03T08:43:42.506 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-05-03T08:43:42.506 INFO:teuthology.packaging:ref: None 2024-05-03T08:43:42.506 INFO:teuthology.packaging:tag: None 2024-05-03T08:43:42.506 INFO:teuthology.packaging:branch: wip-guits-testing-2024-05-02-0628 2024-05-03T08:43:42.506 INFO:teuthology.packaging:sha1: f8e6d2d721bc96a93915917c59bc3d287d57263d 2024-05-03T08:43:42.506 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-guits-testing-2024-05-02-0628 2024-05-03T08:43:42.651 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-3353.gf8e6d2d7 2024-05-03T08:43:42.653 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-05-03T08:43:42.660 INFO:teuthology.task.internal:no buildpackages task found 2024-05-03T08:43:42.660 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-05-03T08:43:42.682 INFO:teuthology.task.internal:Saving configuration 2024-05-03T08:43:42.696 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-05-03T08:43:42.702 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-05-03T08:43:42.726 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi032.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-05-03_06:31:58-orch:cephadm-wip-guits-testing-2024-05-02-0628-distro-default-smithi/7687215', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-03 08:38:57.932821', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEYrsD74lBa50O7lCv8p0GIMhm9Xi+BbpVGMVX9pOlxGLVWXvq1FiPFSf/r9uExnjfKzFofHp+R3y0s3PiB/8H0='} 2024-05-03T08:43:42.748 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi123.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-05-03_06:31:58-orch:cephadm-wip-guits-testing-2024-05-02-0628-distro-default-smithi/7687215', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-03 08:38:57.931156', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIZAxocJHzqm+vJfluTD1dI9ijqhNKBoFjLanVcpN0pIto+9rKftW5yCQJ13k8hZ/j02C2n+cH+NEG0lbFvvfNE='} 2024-05-03T08:43:42.748 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-05-03T08:43:42.754 INFO:teuthology.task.internal:roles: ubuntu@smithi032.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-05-03T08:43:42.754 INFO:teuthology.task.internal:roles: ubuntu@smithi123.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-05-03T08:43:42.754 INFO:teuthology.run_tasks:Running task console_log... 2024-05-03T08:43:42.824 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f253899df70>, signals=[15]) 2024-05-03T08:43:42.824 INFO:teuthology.run_tasks:Running task internal.connect... 2024-05-03T08:43:42.830 INFO:teuthology.task.internal:Opening connections... 2024-05-03T08:43:42.830 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi032.front.sepia.ceph.com 2024-05-03T08:43:42.832 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi032.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-03T08:43:42.910 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi123.front.sepia.ceph.com 2024-05-03T08:43:42.911 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi123.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-03T08:43:42.987 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-05-03T08:43:42.995 DEBUG:teuthology.orchestra.run.smithi032:> uname -m 2024-05-03T08:43:43.014 INFO:teuthology.orchestra.run.smithi032.stdout:x86_64 2024-05-03T08:43:43.014 DEBUG:teuthology.orchestra.run.smithi032:> cat /etc/os-release 2024-05-03T08:43:43.071 INFO:teuthology.orchestra.run.smithi032.stdout:NAME="CentOS Stream" 2024-05-03T08:43:43.072 INFO:teuthology.orchestra.run.smithi032.stdout:VERSION="9" 2024-05-03T08:43:43.072 INFO:teuthology.orchestra.run.smithi032.stdout:ID="centos" 2024-05-03T08:43:43.072 INFO:teuthology.orchestra.run.smithi032.stdout:ID_LIKE="rhel fedora" 2024-05-03T08:43:43.072 INFO:teuthology.orchestra.run.smithi032.stdout:VERSION_ID="9" 2024-05-03T08:43:43.072 INFO:teuthology.orchestra.run.smithi032.stdout:PLATFORM_ID="platform:el9" 2024-05-03T08:43:43.072 INFO:teuthology.orchestra.run.smithi032.stdout:PRETTY_NAME="CentOS Stream 9" 2024-05-03T08:43:43.072 INFO:teuthology.orchestra.run.smithi032.stdout:ANSI_COLOR="0;31" 2024-05-03T08:43:43.072 INFO:teuthology.orchestra.run.smithi032.stdout:LOGO="fedora-logo-icon" 2024-05-03T08:43:43.072 INFO:teuthology.orchestra.run.smithi032.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-05-03T08:43:43.072 INFO:teuthology.orchestra.run.smithi032.stdout:HOME_URL="https://centos.org/" 2024-05-03T08:43:43.072 INFO:teuthology.orchestra.run.smithi032.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-05-03T08:43:43.072 INFO:teuthology.orchestra.run.smithi032.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-05-03T08:43:43.073 INFO:teuthology.orchestra.run.smithi032.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-05-03T08:43:43.073 INFO:teuthology.lock.ops:Updating smithi032.front.sepia.ceph.com on lock server 2024-05-03T08:43:43.094 DEBUG:teuthology.orchestra.run.smithi123:> uname -m 2024-05-03T08:43:43.111 INFO:teuthology.orchestra.run.smithi123.stdout:x86_64 2024-05-03T08:43:43.112 DEBUG:teuthology.orchestra.run.smithi123:> cat /etc/os-release 2024-05-03T08:43:43.168 INFO:teuthology.orchestra.run.smithi123.stdout:NAME="CentOS Stream" 2024-05-03T08:43:43.168 INFO:teuthology.orchestra.run.smithi123.stdout:VERSION="9" 2024-05-03T08:43:43.168 INFO:teuthology.orchestra.run.smithi123.stdout:ID="centos" 2024-05-03T08:43:43.169 INFO:teuthology.orchestra.run.smithi123.stdout:ID_LIKE="rhel fedora" 2024-05-03T08:43:43.169 INFO:teuthology.orchestra.run.smithi123.stdout:VERSION_ID="9" 2024-05-03T08:43:43.169 INFO:teuthology.orchestra.run.smithi123.stdout:PLATFORM_ID="platform:el9" 2024-05-03T08:43:43.169 INFO:teuthology.orchestra.run.smithi123.stdout:PRETTY_NAME="CentOS Stream 9" 2024-05-03T08:43:43.169 INFO:teuthology.orchestra.run.smithi123.stdout:ANSI_COLOR="0;31" 2024-05-03T08:43:43.170 INFO:teuthology.orchestra.run.smithi123.stdout:LOGO="fedora-logo-icon" 2024-05-03T08:43:43.170 INFO:teuthology.orchestra.run.smithi123.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-05-03T08:43:43.170 INFO:teuthology.orchestra.run.smithi123.stdout:HOME_URL="https://centos.org/" 2024-05-03T08:43:43.170 INFO:teuthology.orchestra.run.smithi123.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-05-03T08:43:43.170 INFO:teuthology.orchestra.run.smithi123.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-05-03T08:43:43.170 INFO:teuthology.orchestra.run.smithi123.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-05-03T08:43:43.170 INFO:teuthology.lock.ops:Updating smithi123.front.sepia.ceph.com on lock server 2024-05-03T08:43:43.194 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-05-03T08:43:43.202 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-05-03T08:43:43.219 INFO:teuthology.task.internal:Checking for old test directory... 2024-05-03T08:43:43.220 DEBUG:teuthology.orchestra.run.smithi032:> test '!' -e /home/ubuntu/cephtest 2024-05-03T08:43:43.222 DEBUG:teuthology.orchestra.run.smithi123:> test '!' -e /home/ubuntu/cephtest 2024-05-03T08:43:43.237 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-05-03T08:43:43.248 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-05-03T08:43:43.248 DEBUG:teuthology.orchestra.run.smithi032:> test -z $(ls -A /var/lib/ceph) 2024-05-03T08:43:43.279 DEBUG:teuthology.orchestra.run.smithi123:> test -z $(ls -A /var/lib/ceph) 2024-05-03T08:43:43.366 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-05-03T08:43:43.418 INFO:teuthology.run_tasks:Running task kernel... 2024-05-03T08:43:43.434 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-05-03T08:43:43.434 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-05-03T08:43:43.434 DEBUG:teuthology.orchestra.run.smithi032:> test -f /run/.containerenv -o -f /.dockerenv 2024-05-03T08:43:43.435 DEBUG:teuthology.orchestra.run.smithi123:> test -f /run/.containerenv -o -f /.dockerenv 2024-05-03T08:43:43.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-03T08:43:43.451 DEBUG:teuthology.orchestra.run.smithi123:> uname -r 2024-05-03T08:43:43.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-03T08:43:43.452 DEBUG:teuthology.orchestra.run.smithi032:> uname -r 2024-05-03T08:43:43.507 INFO:teuthology.orchestra.run.smithi123.stdout:5.14.0-437.el9.x86_64 2024-05-03T08:43:43.507 INFO:teuthology.task.kernel:Running kernel on smithi123: 5.14.0-437.el9.x86_64 2024-05-03T08:43:43.508 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum install -y kernel 2024-05-03T08:43:43.510 INFO:teuthology.orchestra.run.smithi032.stdout:5.14.0-437.el9.x86_64 2024-05-03T08:43:43.511 INFO:teuthology.task.kernel:Running kernel on smithi032: 5.14.0-437.el9.x86_64 2024-05-03T08:43:43.511 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum install -y kernel 2024-05-03T08:43:45.738 INFO:teuthology.orchestra.run.smithi123.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.0 MB 00:00 2024-05-03T08:43:47.071 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - BaseOS 4.1 MB/s | 8.0 MB 00:01 2024-05-03T08:43:48.322 INFO:teuthology.orchestra.run.smithi123.stdout:CentOS Stream 9 - AppStream 13 MB/s | 19 MB 00:01 2024-05-03T08:43:49.430 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - AppStream 15 MB/s | 19 MB 00:01 2024-05-03T08:43:52.581 INFO:teuthology.orchestra.run.smithi123.stdout:CentOS Stream 9 - CRB 10 MB/s | 6.2 MB 00:00 2024-05-03T08:43:53.965 INFO:teuthology.orchestra.run.smithi123.stdout:CentOS Stream 9 - Extras packages 69 kB/s | 16 kB 00:00 2024-05-03T08:43:54.714 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - CRB 3.9 MB/s | 6.2 MB 00:01 2024-05-03T08:43:55.566 INFO:teuthology.orchestra.run.smithi123.stdout:Extra Packages for Enterprise Linux 14 MB/s | 21 MB 00:01 2024-05-03T08:43:56.415 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 9 - Extras packages 33 kB/s | 16 kB 00:00 2024-05-03T08:44:00.566 INFO:teuthology.orchestra.run.smithi123.stdout:lab-extras 42 kB/s | 1.7 kB 00:00 2024-05-03T08:44:02.405 INFO:teuthology.orchestra.run.smithi123.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-05-03T08:44:02.405 INFO:teuthology.orchestra.run.smithi123.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-05-03T08:44:02.424 INFO:teuthology.orchestra.run.smithi032.stdout:Extra Packages for Enterprise Linux 3.6 MB/s | 21 MB 00:05 2024-05-03T08:44:02.473 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2024-05-03T08:44:02.474 INFO:teuthology.orchestra.run.smithi123.stdout:Nothing to do. 2024-05-03T08:44:02.474 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2024-05-03T08:44:02.584 DEBUG:teuthology.orchestra.run.smithi123:> echo no | sudo yum reinstall kernel || true 2024-05-03T08:44:03.081 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:00:03 ago on Fri 03 May 2024 08:44:00 AM UTC. 2024-05-03T08:44:03.200 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2024-05-03T08:44:03.212 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2024-05-03T08:44:03.212 INFO:teuthology.orchestra.run.smithi123.stdout: Package Architecture Version Repository Size 2024-05-03T08:44:03.212 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2024-05-03T08:44:03.212 INFO:teuthology.orchestra.run.smithi123.stdout:Reinstalling: 2024-05-03T08:44:03.213 INFO:teuthology.orchestra.run.smithi123.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-05-03T08:44:03.213 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-05-03T08:44:03.213 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2024-05-03T08:44:03.213 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2024-05-03T08:44:03.213 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-05-03T08:44:03.213 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 255 k 2024-05-03T08:44:03.213 INFO:teuthology.orchestra.run.smithi123.stdout:Installed size: 0 2024-05-03T08:44:03.213 INFO:teuthology.orchestra.run.smithi123.stderr:Operation aborted. 2024-05-03T08:44:03.279 INFO:teuthology.orchestra.run.smithi123.stdout:Is this ok [y/N]: 2024-05-03T08:44:03.279 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum reinstall -y kernel || true 2024-05-03T08:44:03.777 INFO:teuthology.orchestra.run.smithi123.stdout:Last metadata expiration check: 0:00:03 ago on Fri 03 May 2024 08:44:00 AM UTC. 2024-05-03T08:44:03.892 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2024-05-03T08:44:03.893 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2024-05-03T08:44:03.893 INFO:teuthology.orchestra.run.smithi123.stdout: Package Architecture Version Repository Size 2024-05-03T08:44:03.893 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2024-05-03T08:44:03.894 INFO:teuthology.orchestra.run.smithi123.stdout:Reinstalling: 2024-05-03T08:44:03.894 INFO:teuthology.orchestra.run.smithi123.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-05-03T08:44:03.894 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-05-03T08:44:03.894 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2024-05-03T08:44:03.894 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2024-05-03T08:44:03.894 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-05-03T08:44:03.894 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 255 k 2024-05-03T08:44:03.894 INFO:teuthology.orchestra.run.smithi123.stdout:Installed size: 0 2024-05-03T08:44:03.894 INFO:teuthology.orchestra.run.smithi123.stdout:Downloading Packages: 2024-05-03T08:44:04.568 INFO:teuthology.orchestra.run.smithi123.stdout:kernel-5.14.0-437.el9.x86_64.rpm 404 kB/s | 255 kB 00:00 2024-05-03T08:44:04.568 INFO:teuthology.orchestra.run.smithi123.stdout:-------------------------------------------------------------------------------- 2024-05-03T08:44:04.568 INFO:teuthology.orchestra.run.smithi123.stdout:Total 378 kB/s | 255 kB 00:00 2024-05-03T08:44:04.587 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction check 2024-05-03T08:44:04.769 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction check succeeded. 2024-05-03T08:44:04.769 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction test 2024-05-03T08:44:04.774 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction test succeeded. 2024-05-03T08:44:04.774 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction 2024-05-03T08:44:04.929 INFO:teuthology.orchestra.run.smithi123.stdout: Preparing : 1/1 2024-05-03T08:44:05.010 INFO:teuthology.orchestra.run.smithi123.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-05-03T08:44:05.010 INFO:teuthology.orchestra.run.smithi123.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-05-03T08:44:05.696 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-05-03T08:44:05.696 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-05-03T08:44:06.388 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-05-03T08:44:06.388 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-05-03T08:44:06.388 INFO:teuthology.orchestra.run.smithi123.stdout:Reinstalled: 2024-05-03T08:44:06.389 INFO:teuthology.orchestra.run.smithi123.stdout: kernel-5.14.0-437.el9.x86_64 2024-05-03T08:44:06.389 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-05-03T08:44:06.389 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2024-05-03T08:44:06.564 DEBUG:teuthology.orchestra.run.smithi123:> rpm -q kernel | sort -rV | head -n 1 2024-05-03T08:44:06.610 INFO:teuthology.orchestra.run.smithi123.stdout:kernel-5.14.0-437.el9.x86_64 2024-05-03T08:44:06.610 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-05-03T08:44:06.611 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-05-03T08:44:06.611 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-05-03T08:44:06.611 DEBUG:teuthology.orchestra.run.smithi123:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-05-03T08:44:06.641 INFO:teuthology.orchestra.run.smithi123.stdout:ttyS1 2024-05-03T08:44:06.665 DEBUG:teuthology.parallel:result is None 2024-05-03T08:44:07.624 INFO:teuthology.orchestra.run.smithi032.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-05-03T08:44:09.581 INFO:teuthology.orchestra.run.smithi032.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-05-03T08:44:09.581 INFO:teuthology.orchestra.run.smithi032.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-05-03T08:44:09.677 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-05-03T08:44:09.678 INFO:teuthology.orchestra.run.smithi032.stdout:Nothing to do. 2024-05-03T08:44:09.678 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-05-03T08:44:09.785 DEBUG:teuthology.orchestra.run.smithi032:> echo no | sudo yum reinstall kernel || true 2024-05-03T08:44:10.307 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:00:03 ago on Fri 03 May 2024 08:44:07 AM UTC. 2024-05-03T08:44:10.430 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-05-03T08:44:10.444 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-05-03T08:44:10.445 INFO:teuthology.orchestra.run.smithi032.stdout: Package Architecture Version Repository Size 2024-05-03T08:44:10.445 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-05-03T08:44:10.445 INFO:teuthology.orchestra.run.smithi032.stdout:Reinstalling: 2024-05-03T08:44:10.445 INFO:teuthology.orchestra.run.smithi032.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-05-03T08:44:10.445 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:44:10.445 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-05-03T08:44:10.445 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-05-03T08:44:10.445 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:44:10.445 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 255 k 2024-05-03T08:44:10.445 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 0 2024-05-03T08:44:10.445 INFO:teuthology.orchestra.run.smithi032.stderr:Operation aborted. 2024-05-03T08:44:10.505 INFO:teuthology.orchestra.run.smithi032.stdout:Is this ok [y/N]: 2024-05-03T08:44:10.505 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum reinstall -y kernel || true 2024-05-03T08:44:11.022 INFO:teuthology.orchestra.run.smithi032.stdout:Last metadata expiration check: 0:00:04 ago on Fri 03 May 2024 08:44:07 AM UTC. 2024-05-03T08:44:11.143 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-05-03T08:44:11.144 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-05-03T08:44:11.144 INFO:teuthology.orchestra.run.smithi032.stdout: Package Architecture Version Repository Size 2024-05-03T08:44:11.144 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-05-03T08:44:11.144 INFO:teuthology.orchestra.run.smithi032.stdout:Reinstalling: 2024-05-03T08:44:11.144 INFO:teuthology.orchestra.run.smithi032.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-05-03T08:44:11.144 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:44:11.145 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-05-03T08:44:11.145 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-05-03T08:44:11.145 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:44:11.145 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 255 k 2024-05-03T08:44:11.145 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 0 2024-05-03T08:44:11.145 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-05-03T08:44:11.551 INFO:teuthology.orchestra.run.smithi032.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.9 MB/s | 255 kB 00:00 2024-05-03T08:44:11.552 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-05-03T08:44:11.552 INFO:teuthology.orchestra.run.smithi032.stdout:Total 626 kB/s | 255 kB 00:00 2024-05-03T08:44:11.571 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-05-03T08:44:11.769 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-05-03T08:44:11.769 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-05-03T08:44:11.774 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-05-03T08:44:11.775 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-05-03T08:44:11.938 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-05-03T08:44:11.994 INFO:teuthology.orchestra.run.smithi032.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-05-03T08:44:11.994 INFO:teuthology.orchestra.run.smithi032.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-05-03T08:44:12.802 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-05-03T08:44:12.803 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-05-03T08:44:13.526 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-05-03T08:44:13.526 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:44:13.526 INFO:teuthology.orchestra.run.smithi032.stdout:Reinstalled: 2024-05-03T08:44:13.526 INFO:teuthology.orchestra.run.smithi032.stdout: kernel-5.14.0-437.el9.x86_64 2024-05-03T08:44:13.526 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:44:13.526 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-05-03T08:44:13.708 DEBUG:teuthology.orchestra.run.smithi032:> rpm -q kernel | sort -rV | head -n 1 2024-05-03T08:44:13.756 INFO:teuthology.orchestra.run.smithi032.stdout:kernel-5.14.0-437.el9.x86_64 2024-05-03T08:44:13.756 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-05-03T08:44:13.757 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-05-03T08:44:13.757 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-05-03T08:44:13.757 DEBUG:teuthology.orchestra.run.smithi032:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-05-03T08:44:13.786 INFO:teuthology.orchestra.run.smithi032.stdout:ttyS1 2024-05-03T08:44:13.812 DEBUG:teuthology.parallel:result is None 2024-05-03T08:44:13.812 INFO:teuthology.run_tasks:Running task internal.base... 2024-05-03T08:44:13.823 INFO:teuthology.task.internal:Creating test directory... 2024-05-03T08:44:13.824 DEBUG:teuthology.orchestra.run.smithi032:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-05-03T08:44:13.855 DEBUG:teuthology.orchestra.run.smithi123:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-05-03T08:44:13.873 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-05-03T08:44:13.880 INFO:teuthology.run_tasks:Running task internal.archive... 2024-05-03T08:44:13.897 INFO:teuthology.task.internal:Creating archive directory... 2024-05-03T08:44:13.897 DEBUG:teuthology.orchestra.run.smithi032:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-05-03T08:44:13.910 DEBUG:teuthology.orchestra.run.smithi123:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-05-03T08:44:14.011 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-05-03T08:44:14.020 INFO:teuthology.task.internal:Enabling coredump saving... 2024-05-03T08:44:14.020 DEBUG:teuthology.orchestra.run.smithi032:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-05-03T08:44:14.022 DEBUG:teuthology.orchestra.run.smithi123:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-05-03T08:44:14.051 INFO:teuthology.orchestra.run.smithi032.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-03T08:44:14.061 INFO:teuthology.orchestra.run.smithi032.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-03T08:44:14.080 INFO:teuthology.orchestra.run.smithi123.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-03T08:44:14.090 INFO:teuthology.orchestra.run.smithi123.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-03T08:44:14.092 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-05-03T08:44:14.100 INFO:teuthology.task.internal:Configuring sudo... 2024-05-03T08:44:14.101 DEBUG:teuthology.orchestra.run.smithi032:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-05-03T08:44:14.105 DEBUG:teuthology.orchestra.run.smithi123:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-05-03T08:44:14.131 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-05-03T08:44:14.239 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-05-03T08:44:14.239 DEBUG:teuthology.orchestra.run.smithi032:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-05-03T08:44:14.241 DEBUG:teuthology.orchestra.run.smithi123:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-05-03T08:44:14.258 DEBUG:teuthology.orchestra.run.smithi032:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-03T08:44:14.335 DEBUG:teuthology.orchestra.run.smithi032:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-03T08:44:14.387 DEBUG:teuthology.orchestra.run.smithi032:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-03T08:44:14.470 DEBUG:teuthology.orchestra.run.smithi032:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-03T08:44:14.508 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-05-03T08:44:14.508 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-05-03T08:44:14.572 DEBUG:teuthology.orchestra.run.smithi123:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-03T08:44:14.616 DEBUG:teuthology.orchestra.run.smithi123:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-03T08:44:14.667 DEBUG:teuthology.orchestra.run.smithi123:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-03T08:44:14.751 DEBUG:teuthology.orchestra.run.smithi123:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-03T08:44:14.795 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-05-03T08:44:14.795 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-05-03T08:44:14.863 DEBUG:teuthology.orchestra.run.smithi032:> sudo service rsyslog restart 2024-05-03T08:44:14.865 DEBUG:teuthology.orchestra.run.smithi123:> sudo service rsyslog restart 2024-05-03T08:44:14.898 INFO:teuthology.orchestra.run.smithi032.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-03T08:44:14.951 INFO:teuthology.orchestra.run.smithi123.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-03T08:44:15.268 INFO:teuthology.run_tasks:Running task internal.timer... 2024-05-03T08:44:15.276 INFO:teuthology.task.internal:Starting timer... 2024-05-03T08:44:15.276 INFO:teuthology.run_tasks:Running task pcp... 2024-05-03T08:44:15.314 INFO:teuthology.run_tasks:Running task selinux... 2024-05-03T08:44:15.356 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-05-03T08:44:15.357 DEBUG:teuthology.orchestra.run.smithi032:> sudo service auditd rotate 2024-05-03T08:44:15.406 INFO:teuthology.orchestra.run.smithi032.stdout:Rotating logs: 2024-05-03T08:44:15.408 DEBUG:teuthology.orchestra.run.smithi123:> sudo service auditd rotate 2024-05-03T08:44:15.459 INFO:teuthology.orchestra.run.smithi123.stdout:Rotating logs: 2024-05-03T08:44:15.461 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-05-03T08:44:15.461 DEBUG:teuthology.orchestra.run.smithi032:> /usr/sbin/getenforce 2024-05-03T08:44:15.489 INFO:teuthology.orchestra.run.smithi032.stdout:Permissive 2024-05-03T08:44:15.489 DEBUG:teuthology.orchestra.run.smithi123:> /usr/sbin/getenforce 2024-05-03T08:44:15.508 INFO:teuthology.orchestra.run.smithi123.stdout:Permissive 2024-05-03T08:44:15.508 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi032.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi123.front.sepia.ceph.com': 'permissive'} 2024-05-03T08:44:15.508 DEBUG:teuthology.orchestra.run.smithi032:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-05-03T08:44:15.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-03T08:44:15.554 DEBUG:teuthology.orchestra.run.smithi123:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-05-03T08:44:15.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-03T08:44:15.579 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-05-03T08:44:15.580 DEBUG:teuthology.orchestra.run.smithi032:> sudo /usr/sbin/setenforce permissive 2024-05-03T08:44:15.624 DEBUG:teuthology.orchestra.run.smithi123:> sudo /usr/sbin/setenforce permissive 2024-05-03T08:44:15.648 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-05-03T08:44:15.658 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-05-03T08:44:15.664 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-05-03T08:44:15.794 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-05-03T08:44:15.809 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-05-03T08:44:15.810 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi032.front.sepia.ceph.com,smithi123.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-05-03T08:48:11.224 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi032.front.sepia.ceph.com'), Remote(name='ubuntu@smithi123.front.sepia.ceph.com')] 2024-05-03T08:48:11.225 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi032.front.sepia.ceph.com' 2024-05-03T08:48:11.227 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi032.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-03T08:48:11.303 DEBUG:teuthology.orchestra.run.smithi032:> true 2024-05-03T08:48:11.390 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi032.front.sepia.ceph.com' 2024-05-03T08:48:11.390 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi123.front.sepia.ceph.com' 2024-05-03T08:48:11.391 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi123.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-03T08:48:11.463 DEBUG:teuthology.orchestra.run.smithi123:> true 2024-05-03T08:48:11.497 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi123.front.sepia.ceph.com' 2024-05-03T08:48:11.498 INFO:teuthology.run_tasks:Running task clock... 2024-05-03T08:48:11.509 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-05-03T08:48:11.509 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-05-03T08:48:11.510 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-03T08:48:11.512 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-05-03T08:48:11.512 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-03T08:48:11.539 INFO:teuthology.orchestra.run.smithi032.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-05-03T08:48:11.553 INFO:teuthology.orchestra.run.smithi032.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-05-03T08:48:11.564 INFO:teuthology.orchestra.run.smithi123.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-05-03T08:48:11.577 INFO:teuthology.orchestra.run.smithi123.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-05-03T08:48:11.578 INFO:teuthology.orchestra.run.smithi032.stderr:sudo: ntpd: command not found 2024-05-03T08:48:11.589 INFO:teuthology.orchestra.run.smithi032.stdout:506 Cannot talk to daemon 2024-05-03T08:48:11.599 INFO:teuthology.orchestra.run.smithi123.stderr:sudo: ntpd: command not found 2024-05-03T08:48:11.602 INFO:teuthology.orchestra.run.smithi032.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-05-03T08:48:11.609 INFO:teuthology.orchestra.run.smithi123.stdout:506 Cannot talk to daemon 2024-05-03T08:48:11.616 INFO:teuthology.orchestra.run.smithi032.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-05-03T08:48:11.622 INFO:teuthology.orchestra.run.smithi123.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-05-03T08:48:11.635 INFO:teuthology.orchestra.run.smithi123.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-05-03T08:48:11.669 INFO:teuthology.orchestra.run.smithi032.stderr:bash: line 1: ntpq: command not found 2024-05-03T08:48:11.673 INFO:teuthology.orchestra.run.smithi032.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-03T08:48:11.673 INFO:teuthology.orchestra.run.smithi032.stdout:=============================================================================== 2024-05-03T08:48:11.673 INFO:teuthology.orchestra.run.smithi032.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-03T08:48:11.673 INFO:teuthology.orchestra.run.smithi032.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-03T08:48:11.673 INFO:teuthology.orchestra.run.smithi032.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-03T08:48:11.674 INFO:teuthology.orchestra.run.smithi032.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-03T08:48:11.691 INFO:teuthology.orchestra.run.smithi123.stderr:bash: line 1: ntpq: command not found 2024-05-03T08:48:11.694 INFO:teuthology.orchestra.run.smithi123.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-03T08:48:11.695 INFO:teuthology.orchestra.run.smithi123.stdout:=============================================================================== 2024-05-03T08:48:11.695 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-03T08:48:11.695 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-03T08:48:11.695 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-03T08:48:11.695 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-03T08:48:11.695 INFO:teuthology.run_tasks:Running task cephadm... 2024-05-03T08:48:11.794 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': 'f8e6d2d721bc96a93915917c59bc3d287d57263d'} 2024-05-03T08:48:11.795 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-05-03T08:48:11.795 INFO:tasks.cephadm:Cluster fsid is df63fec2-0929-11ef-bc97-c7b262605968 2024-05-03T08:48:11.795 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-05-03T08:48:11.795 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-05-03T08:48:11.796 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi032': '172.21.15.32', 'mon.smithi123': '172.21.15.123'} 2024-05-03T08:48:11.796 INFO:tasks.cephadm:Normalizing hostnames... 2024-05-03T08:48:11.796 DEBUG:teuthology.orchestra.run.smithi032:> sudo hostname $(hostname -s) 2024-05-03T08:48:11.822 DEBUG:teuthology.orchestra.run.smithi123:> sudo hostname $(hostname -s) 2024-05-03T08:48:11.845 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-05-03T08:48:11.846 DEBUG:teuthology.orchestra.run.smithi032:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-05-03T08:48:12.140 INFO:teuthology.orchestra.run.smithi032.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 May 3 08:48 /home/ubuntu/cephtest/cephadm 2024-05-03T08:48:12.141 DEBUG:teuthology.orchestra.run.smithi123:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-05-03T08:48:12.295 INFO:teuthology.orchestra.run.smithi123.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 May 3 08:48 /home/ubuntu/cephtest/cephadm 2024-05-03T08:48:12.296 DEBUG:teuthology.orchestra.run.smithi032:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-05-03T08:48:12.312 DEBUG:teuthology.orchestra.run.smithi123:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-05-03T08:48:12.336 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-05-03T08:48:12.336 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-05-03T08:48:12.355 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-05-03T08:48:13.920 INFO:teuthology.orchestra.run.smithi032.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-03T08:48:13.954 INFO:teuthology.orchestra.run.smithi123.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-03T08:48:31.850 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-05-03T08:48:31.850 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-05-03T08:48:31.851 INFO:teuthology.orchestra.run.smithi123.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-05-03T08:48:31.851 INFO:teuthology.orchestra.run.smithi123.stdout: "repo_digests": [ 2024-05-03T08:48:31.851 INFO:teuthology.orchestra.run.smithi123.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-05-03T08:48:31.851 INFO:teuthology.orchestra.run.smithi123.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-05-03T08:48:31.851 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2024-05-03T08:48:31.851 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-05-03T08:48:37.389 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-05-03T08:48:37.389 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-05-03T08:48:37.389 INFO:teuthology.orchestra.run.smithi032.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-05-03T08:48:37.389 INFO:teuthology.orchestra.run.smithi032.stdout: "repo_digests": [ 2024-05-03T08:48:37.389 INFO:teuthology.orchestra.run.smithi032.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-05-03T08:48:37.389 INFO:teuthology.orchestra.run.smithi032.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-05-03T08:48:37.390 INFO:teuthology.orchestra.run.smithi032.stdout: ] 2024-05-03T08:48:37.390 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-05-03T08:48:37.408 DEBUG:teuthology.orchestra.run.smithi032:> sudo mkdir -p /etc/ceph 2024-05-03T08:48:37.465 DEBUG:teuthology.orchestra.run.smithi123:> sudo mkdir -p /etc/ceph 2024-05-03T08:48:37.497 DEBUG:teuthology.orchestra.run.smithi032:> sudo chmod 777 /etc/ceph 2024-05-03T08:48:37.522 DEBUG:teuthology.orchestra.run.smithi123:> sudo chmod 777 /etc/ceph 2024-05-03T08:48:37.565 INFO:tasks.cephadm:Writing seed config... 2024-05-03T08:48:37.566 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-05-03T08:48:37.566 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-05-03T08:48:37.566 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-05-03T08:48:37.566 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-05-03T08:48:37.566 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-05-03T08:48:37.566 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-05-03T08:48:37.567 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-05-03T08:48:37.567 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-05-03T08:48:37.567 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-05-03T08:48:37.567 DEBUG:teuthology.orchestra.run.smithi032:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-05-03T08:48:37.583 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = df63fec2-0929-11ef-bc97-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-05-03T08:48:37.583 DEBUG:teuthology.orchestra.run.smithi032:mon.smithi032> sudo journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@mon.smithi032.service 2024-05-03T08:48:37.625 INFO:tasks.cephadm:Bootstrapping... 2024-05-03T08:48:37.625 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid df63fec2-0929-11ef-bc97-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.32 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-05-03T08:48:37.823 INFO:teuthology.orchestra.run.smithi032.stderr:-------------------------------------------------------------------------------- 2024-05-03T08:48:37.823 INFO:teuthology.orchestra.run.smithi032.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'df63fec2-0929-11ef-bc97-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.32', '--skip-admin-label'] 2024-05-03T08:48:37.847 INFO:teuthology.orchestra.run.smithi032.stderr:/bin/podman: 4.9.4-dev 2024-05-03T08:48:37.849 INFO:teuthology.orchestra.run.smithi032.stderr:Verifying podman|docker is present... 2024-05-03T08:48:37.871 INFO:teuthology.orchestra.run.smithi032.stderr:/bin/podman: 4.9.4-dev 2024-05-03T08:48:37.873 INFO:teuthology.orchestra.run.smithi032.stderr:Verifying lvm2 is present... 2024-05-03T08:48:37.873 INFO:teuthology.orchestra.run.smithi032.stderr:Verifying time synchronization is in place... 2024-05-03T08:48:37.880 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-05-03T08:48:37.886 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: inactive 2024-05-03T08:48:37.893 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: enabled 2024-05-03T08:48:37.899 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: active 2024-05-03T08:48:37.900 INFO:teuthology.orchestra.run.smithi032.stderr:Unit chronyd.service is enabled and running 2024-05-03T08:48:37.900 INFO:teuthology.orchestra.run.smithi032.stderr:Repeating the final host check... 2024-05-03T08:48:37.921 INFO:teuthology.orchestra.run.smithi032.stderr:/bin/podman: 4.9.4-dev 2024-05-03T08:48:37.923 INFO:teuthology.orchestra.run.smithi032.stderr:podman (/bin/podman) version 4.9.4 is present 2024-05-03T08:48:37.924 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl is present 2024-05-03T08:48:37.924 INFO:teuthology.orchestra.run.smithi032.stderr:lvcreate is present 2024-05-03T08:48:37.930 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-05-03T08:48:37.936 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: inactive 2024-05-03T08:48:37.943 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: enabled 2024-05-03T08:48:37.949 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: active 2024-05-03T08:48:37.950 INFO:teuthology.orchestra.run.smithi032.stderr:Unit chronyd.service is enabled and running 2024-05-03T08:48:37.950 INFO:teuthology.orchestra.run.smithi032.stderr:Host looks OK 2024-05-03T08:48:37.950 INFO:teuthology.orchestra.run.smithi032.stderr:Cluster fsid: df63fec2-0929-11ef-bc97-c7b262605968 2024-05-03T08:48:37.950 INFO:teuthology.orchestra.run.smithi032.stderr:Acquiring lock 139692026102496 on /run/cephadm/df63fec2-0929-11ef-bc97-c7b262605968.lock 2024-05-03T08:48:37.950 INFO:teuthology.orchestra.run.smithi032.stderr:Lock 139692026102496 acquired on /run/cephadm/df63fec2-0929-11ef-bc97-c7b262605968.lock 2024-05-03T08:48:37.951 INFO:teuthology.orchestra.run.smithi032.stderr:Verifying IP 172.21.15.32 port 3300 ... 2024-05-03T08:48:37.951 INFO:teuthology.orchestra.run.smithi032.stderr:Verifying IP 172.21.15.32 port 6789 ... 2024-05-03T08:48:37.951 INFO:teuthology.orchestra.run.smithi032.stderr:Base mon IP is 172.21.15.32, final addrv is [v2:172.21.15.32:3300,v1:172.21.15.32:6789] 2024-05-03T08:48:37.953 INFO:teuthology.orchestra.run.smithi032.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.32 metric 100 2024-05-03T08:48:37.953 INFO:teuthology.orchestra.run.smithi032.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.32 metric 100 2024-05-03T08:48:37.956 INFO:teuthology.orchestra.run.smithi032.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-05-03T08:48:37.956 INFO:teuthology.orchestra.run.smithi032.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-05-03T08:48:37.956 INFO:teuthology.orchestra.run.smithi032.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-05-03T08:48:37.958 INFO:teuthology.orchestra.run.smithi032.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-05-03T08:48:37.958 INFO:teuthology.orchestra.run.smithi032.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-05-03T08:48:37.959 INFO:teuthology.orchestra.run.smithi032.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-05-03T08:48:37.959 INFO:teuthology.orchestra.run.smithi032.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-05-03T08:48:37.959 INFO:teuthology.orchestra.run.smithi032.stderr:/sbin/ip: inet6 fe80::ae1f:6bff:fef8:2602/64 scope link noprefixroute 2024-05-03T08:48:37.959 INFO:teuthology.orchestra.run.smithi032.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-05-03T08:48:37.959 INFO:teuthology.orchestra.run.smithi032.stderr:Mon IP `172.21.15.32` is in CIDR network `172.21.0.0/20` 2024-05-03T08:48:37.960 INFO:teuthology.orchestra.run.smithi032.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-05-03T08:48:37.960 INFO:teuthology.orchestra.run.smithi032.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-03T08:48:38.066 INFO:teuthology.orchestra.run.smithi032.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-05-03T08:48:38.422 INFO:teuthology.orchestra.run.smithi032.stderr:/bin/podman: Getting image source signatures 2024-05-03T08:48:38.422 INFO:teuthology.orchestra.run.smithi032.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-05-03T08:48:38.422 INFO:teuthology.orchestra.run.smithi032.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-05-03T08:48:38.422 INFO:teuthology.orchestra.run.smithi032.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-05-03T08:48:38.422 INFO:teuthology.orchestra.run.smithi032.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-05-03T08:48:38.422 INFO:teuthology.orchestra.run.smithi032.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-05-03T08:48:38.422 INFO:teuthology.orchestra.run.smithi032.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-05-03T08:48:38.710 INFO:teuthology.orchestra.run.smithi032.stderr:/bin/podman: Writing manifest to image destination 2024-05-03T08:48:38.861 INFO:teuthology.orchestra.run.smithi032.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-05-03T08:48:39.280 INFO:teuthology.orchestra.run.smithi032.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-05-03T08:48:39.770 INFO:teuthology.orchestra.run.smithi032.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-05-03T08:48:39.770 INFO:teuthology.orchestra.run.smithi032.stderr:Extracting ceph user uid/gid from container image... 2024-05-03T08:48:40.194 INFO:teuthology.orchestra.run.smithi032.stderr:stat: 167 167 2024-05-03T08:48:40.685 INFO:teuthology.orchestra.run.smithi032.stderr:Creating initial keys... 2024-05-03T08:48:41.075 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph-authtool: AQDppDRm/WUGABAAh+G8ra5WYAF3tYTq6EYswg== 2024-05-03T08:48:42.007 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph-authtool: AQDppDRmutJ8NxAAEoqP6M5hlvvwUVeHt9GCqg== 2024-05-03T08:48:42.887 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph-authtool: AQDqpDRmrbzeLxAA5bV02dakdm/wdR3KZTrudA== 2024-05-03T08:48:43.305 INFO:teuthology.orchestra.run.smithi032.stderr:Creating initial monmap... 2024-05-03T08:48:43.704 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-05-03T08:48:43.704 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-05-03T08:48:43.704 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to df63fec2-0929-11ef-bc97-c7b262605968 2024-05-03T08:48:43.704 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-05-03T08:48:44.193 INFO:teuthology.orchestra.run.smithi032.stderr:monmaptool for smithi032 [v2:172.21.15.32:3300,v1:172.21.15.32:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-05-03T08:48:44.194 INFO:teuthology.orchestra.run.smithi032.stderr:setting min_mon_release = octopus 2024-05-03T08:48:44.194 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/monmaptool: set fsid to df63fec2-0929-11ef-bc97-c7b262605968 2024-05-03T08:48:44.194 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-05-03T08:48:44.194 INFO:teuthology.orchestra.run.smithi032.stderr: 2024-05-03T08:48:44.194 INFO:teuthology.orchestra.run.smithi032.stderr:Creating mon... 2024-05-03T08:48:44.975 INFO:teuthology.orchestra.run.smithi032.stderr:create mon.smithi032 on 2024-05-03T08:48:45.185 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-05-03T08:48:45.365 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-df63fec2-0929-11ef-bc97-c7b262605968.target → /etc/systemd/system/ceph-df63fec2-0929-11ef-bc97-c7b262605968.target. 2024-05-03T08:48:45.366 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-df63fec2-0929-11ef-bc97-c7b262605968.target → /etc/systemd/system/ceph-df63fec2-0929-11ef-bc97-c7b262605968.target. 2024-05-03T08:48:45.682 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: Failed to reset failed state of unit ceph-df63fec2-0929-11ef-bc97-c7b262605968@mon.smithi032.service: Unit ceph-df63fec2-0929-11ef-bc97-c7b262605968@mon.smithi032.service not loaded. 2024-05-03T08:48:45.690 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: Created symlink /etc/systemd/system/ceph-df63fec2-0929-11ef-bc97-c7b262605968.target.wants/ceph-df63fec2-0929-11ef-bc97-c7b262605968@mon.smithi032.service → /etc/systemd/system/ceph-df63fec2-0929-11ef-bc97-c7b262605968@.service. 2024-05-03T08:48:46.383 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:46 smithi032 ceph-mon[22060]: mon.smithi032 is new leader, mons smithi032 in quorum (ranks 0) 2024-05-03T08:48:46.396 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: disabled 2024-05-03T08:48:46.402 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: inactive 2024-05-03T08:48:46.403 INFO:teuthology.orchestra.run.smithi032.stderr:firewalld.service is not enabled 2024-05-03T08:48:46.403 INFO:teuthology.orchestra.run.smithi032.stderr:Not possible to enable service . firewalld.service is not available 2024-05-03T08:48:46.404 INFO:teuthology.orchestra.run.smithi032.stderr:Waiting for mon to start... 2024-05-03T08:48:46.404 INFO:teuthology.orchestra.run.smithi032.stderr:Waiting for mon... 2024-05-03T08:48:46.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:46 smithi032 bash[22042]: b595b2d1a37ac4c6ca145da54b5db51a11685fa543e022b173a578fad2010fa1 2024-05-03T08:48:46.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:46 smithi032 systemd[1]: Started Ceph mon.smithi032 for df63fec2-0929-11ef-bc97-c7b262605968. 2024-05-03T08:48:46.922 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: cluster: 2024-05-03T08:48:46.922 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: id: df63fec2-0929-11ef-bc97-c7b262605968 2024-05-03T08:48:46.923 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-05-03T08:48:46.923 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: 2024-05-03T08:48:46.923 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: services: 2024-05-03T08:48:46.923 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi032 (age 0.607921s) 2024-05-03T08:48:46.923 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: mgr: no daemons active 2024-05-03T08:48:46.923 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-05-03T08:48:46.923 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: 2024-05-03T08:48:46.923 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: data: 2024-05-03T08:48:46.923 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-05-03T08:48:46.923 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-05-03T08:48:46.923 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-05-03T08:48:46.923 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: pgs: 2024-05-03T08:48:46.923 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: 2024-05-03T08:48:47.326 INFO:teuthology.orchestra.run.smithi032.stderr:mon is available 2024-05-03T08:48:47.326 INFO:teuthology.orchestra.run.smithi032.stderr:Assimilating anything we can from ceph.conf... 2024-05-03T08:48:47.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:47 smithi032 ceph-mon[22060]: mon.smithi032 is new leader, mons smithi032 in quorum (ranks 0) 2024-05-03T08:48:47.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:47 smithi032 ceph-mon[22060]: monmap e1: 1 mons at {smithi032=[v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0]} 2024-05-03T08:48:47.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:47 smithi032 ceph-mon[22060]: fsmap 2024-05-03T08:48:47.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:47 smithi032 ceph-mon[22060]: osdmap e1: 0 total, 0 up, 0 in 2024-05-03T08:48:47.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:47 smithi032 ceph-mon[22060]: mgrmap e1: no daemons active 2024-05-03T08:48:47.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:47 smithi032 ceph-mon[22060]: from='client.? 172.21.15.32:0/1968528951' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-03T08:48:47.856 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: 2024-05-03T08:48:47.856 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: [global] 2024-05-03T08:48:47.856 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: fsid = df63fec2-0929-11ef-bc97-c7b262605968 2024-05-03T08:48:47.856 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.32:3300,v1:172.21.15.32:6789] 2024-05-03T08:48:47.856 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-05-03T08:48:47.856 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-05-03T08:48:47.856 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-05-03T08:48:47.856 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-05-03T08:48:47.857 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: 2024-05-03T08:48:47.857 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: [mgr] 2024-05-03T08:48:47.857 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-05-03T08:48:47.857 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: 2024-05-03T08:48:47.857 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: [osd] 2024-05-03T08:48:47.857 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-05-03T08:48:47.857 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-05-03T08:48:48.407 INFO:teuthology.orchestra.run.smithi032.stderr:Generating new minimal ceph.conf... 2024-05-03T08:48:48.632 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:48 smithi032 ceph-mon[22060]: from='client.? 172.21.15.32:0/961951115' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-05-03T08:48:48.632 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:48 smithi032 ceph-mon[22060]: from='client.? 172.21.15.32:0/961951115' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-05-03T08:48:49.439 INFO:teuthology.orchestra.run.smithi032.stderr:Restarting the monitor... 2024-05-03T08:48:49.589 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:49 smithi032 ceph-mon[22060]: from='client.? 172.21.15.32:0/1034678712' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:48:49.589 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:49 smithi032 systemd[1]: Stopping Ceph mon.smithi032 for df63fec2-0929-11ef-bc97-c7b262605968... 2024-05-03T08:48:49.589 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:49 smithi032 bash[22277]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon.smithi032" found: no such container 2024-05-03T08:48:49.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:49 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi032[22056]: 2024-05-03T08:48:49.587+0000 7fc62bdb9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi032 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-03T08:48:49.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:49 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi032[22056]: 2024-05-03T08:48:49.587+0000 7fc62bdb9700 -1 mon.smithi032@0(leader) e1 *** Got Signal Terminated *** 2024-05-03T08:48:49.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:49 smithi032 podman[22288]: 2024-05-03 08:48:49.751842467 +0000 UTC m=+0.255500296 container died b595b2d1a37ac4c6ca145da54b5db51a11685fa543e022b173a578fad2010fa1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi032, version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, GIT_CLEAN=True, io.openshift.expose-services=, name=centos-stream, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , vcs-type=git) 2024-05-03T08:48:50.281 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 podman[22288]: 2024-05-03 08:48:50.020664644 +0000 UTC m=+0.524322476 container cleanup b595b2d1a37ac4c6ca145da54b5db51a11685fa543e022b173a578fad2010fa1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi032, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, release=754, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, ceph=True, name=centos-stream, maintainer=Guillaume Abrioux , vcs-type=git) 2024-05-03T08:48:50.281 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 bash[22288]: ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi032 2024-05-03T08:48:50.281 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 bash[22314]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon.smithi032" found: no such container 2024-05-03T08:48:50.587 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 podman[22302]: 2024-05-03 08:48:50.279977404 +0000 UTC m=+0.524515776 container remove b595b2d1a37ac4c6ca145da54b5db51a11685fa543e022b173a578fad2010fa1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi032, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.openshift.expose-services=, architecture=x86_64, com.redhat.component=centos-stream-container, RELEASE=HEAD, maintainer=Guillaume Abrioux , vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-03T08:48:50.587 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 systemd[1]: ceph-df63fec2-0929-11ef-bc97-c7b262605968@mon.smithi032.service: Deactivated successfully. 2024-05-03T08:48:50.587 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 systemd[1]: Stopped Ceph mon.smithi032 for df63fec2-0929-11ef-bc97-c7b262605968. 2024-05-03T08:48:50.587 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 systemd[1]: Starting Ceph mon.smithi032 for df63fec2-0929-11ef-bc97-c7b262605968... 2024-05-03T08:48:50.587 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 podman[22384]: 2024-05-03 08:48:50.505951069 +0000 UTC m=+0.098044071 container create 7e5da84a501eb3ff7337cde84c7f3667e36b6b4ce6f199f91b9b974421ca3ff8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi032, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, version=8, release=754, io.openshift.tags=base centos centos-stream, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_CLEAN=True, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-05-03T08:48:50.587 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 podman[22384]: 2024-05-03 08:48:50.427364782 +0000 UTC m=+0.019457777 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-05-03T08:48:50.737 INFO:teuthology.orchestra.run.smithi032.stderr:Setting mon public_network to 172.21.0.0/20 2024-05-03T08:48:50.860 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 podman[22384]: 2024-05-03 08:48:50.658548043 +0000 UTC m=+0.250641021 container init 7e5da84a501eb3ff7337cde84c7f3667e36b6b4ce6f199f91b9b974421ca3ff8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi032, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, vcs-type=git, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, name=centos-stream, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-05-03T08:48:50.860 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 podman[22384]: 2024-05-03 08:48:50.66172478 +0000 UTC m=+0.253817740 container start 7e5da84a501eb3ff7337cde84c7f3667e36b6b4ce6f199f91b9b974421ca3ff8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi032, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, ceph=True, vcs-type=git, io.openshift.expose-services=, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD) 2024-05-03T08:48:50.861 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: set uid:gid to 167:167 (ceph:ceph) 2024-05-03T08:48:50.861 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-05-03T08:48:50.861 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: pidfile_write: ignore empty --pid-file 2024-05-03T08:48:50.861 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: load: jerasure load: lrc 2024-05-03T08:48:50.861 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: RocksDB version: 6.15.5 2024-05-03T08:48:50.862 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-05-03T08:48:50.862 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Compile date Apr 18 2022 2024-05-03T08:48:50.862 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: DB SUMMARY 2024-05-03T08:48:50.862 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: DB Session ID: MVZ9TRZ1JGC7LUC3RESX 2024-05-03T08:48:50.862 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: CURRENT file: CURRENT 2024-05-03T08:48:50.862 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: IDENTITY file: IDENTITY 2024-05-03T08:48:50.862 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-05-03T08:48:50.862 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi032/store.db dir, Total Num: 1, files: 000008.sst 2024-05-03T08:48:50.862 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi032/store.db: 000010.log size: 82705 ; 2024-05-03T08:48:50.862 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.error_if_exists: 0 2024-05-03T08:48:50.862 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.create_if_missing: 0 2024-05-03T08:48:50.863 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.paranoid_checks: 1 2024-05-03T08:48:50.863 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-03T08:48:50.863 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.env: 0x557b61cc6860 2024-05-03T08:48:50.863 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.fs: Posix File System 2024-05-03T08:48:50.863 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.info_log: 0x557b62b61fa0 2024-05-03T08:48:50.863 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_file_opening_threads: 16 2024-05-03T08:48:50.863 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.statistics: (nil) 2024-05-03T08:48:50.863 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.use_fsync: 0 2024-05-03T08:48:50.863 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_log_file_size: 0 2024-05-03T08:48:50.863 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-03T08:48:50.863 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-03T08:48:50.863 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.keep_log_file_num: 1000 2024-05-03T08:48:50.863 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.recycle_log_file_num: 0 2024-05-03T08:48:50.863 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.allow_fallocate: 1 2024-05-03T08:48:50.864 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.allow_mmap_reads: 0 2024-05-03T08:48:50.864 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.allow_mmap_writes: 0 2024-05-03T08:48:50.864 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.use_direct_reads: 0 2024-05-03T08:48:50.864 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-03T08:48:50.864 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.create_missing_column_families: 0 2024-05-03T08:48:50.864 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.db_log_dir: 2024-05-03T08:48:50.864 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi032/store.db 2024-05-03T08:48:50.864 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-03T08:48:50.864 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-03T08:48:50.864 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-03T08:48:50.864 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-03T08:48:50.864 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-03T08:48:50.864 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-03T08:48:50.864 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.advise_random_on_open: 1 2024-05-03T08:48:50.864 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.db_write_buffer_size: 0 2024-05-03T08:48:50.865 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.write_buffer_manager: 0x557b62c52330 2024-05-03T08:48:50.865 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-03T08:48:50.865 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-05-03T08:48:50.865 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-03T08:48:50.865 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-03T08:48:50.865 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.rate_limiter: (nil) 2024-05-03T08:48:50.865 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-03T08:48:50.865 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.wal_recovery_mode: 2 2024-05-03T08:48:50.865 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.enable_thread_tracking: 0 2024-05-03T08:48:50.865 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.enable_pipelined_write: 0 2024-05-03T08:48:50.865 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.unordered_write: 0 2024-05-03T08:48:50.865 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-03T08:48:50.865 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-03T08:48:50.865 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-03T08:48:50.865 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-03T08:48:50.866 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.row_cache: None 2024-05-03T08:48:50.866 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.wal_filter: None 2024-05-03T08:48:50.866 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-03T08:48:50.866 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.allow_ingest_behind: 0 2024-05-03T08:48:50.866 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.preserve_deletes: 0 2024-05-03T08:48:50.866 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.two_write_queues: 0 2024-05-03T08:48:50.866 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.manual_wal_flush: 0 2024-05-03T08:48:50.866 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.atomic_flush: 0 2024-05-03T08:48:50.866 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-03T08:48:50.866 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-03T08:48:50.866 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-03T08:48:50.866 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.log_readahead_size: 0 2024-05-03T08:48:50.866 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-03T08:48:50.866 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.best_efforts_recovery: 0 2024-05-03T08:48:50.867 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-03T08:48:50.867 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-03T08:48:50.867 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.allow_data_in_errors: 0 2024-05-03T08:48:50.867 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.db_host_id: __hostname__ 2024-05-03T08:48:50.867 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_background_jobs: 2 2024-05-03T08:48:50.867 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_background_compactions: -1 2024-05-03T08:48:50.867 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_subcompactions: 1 2024-05-03T08:48:50.867 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-03T08:48:50.867 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-03T08:48:50.867 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-03T08:48:50.867 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_total_wal_size: 0 2024-05-03T08:48:50.867 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-03T08:48:50.867 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-03T08:48:50.867 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-03T08:48:50.867 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-03T08:48:50.868 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_open_files: -1 2024-05-03T08:48:50.868 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.bytes_per_sync: 0 2024-05-03T08:48:50.868 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-03T08:48:50.868 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-03T08:48:50.868 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compaction_readahead_size: 0 2024-05-03T08:48:50.868 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_background_flushes: -1 2024-05-03T08:48:50.868 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Compression algorithms supported: 2024-05-03T08:48:50.868 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-03T08:48:50.868 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: kZSTD supported: 0 2024-05-03T08:48:50.868 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: kXpressCompression supported: 0 2024-05-03T08:48:50.868 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: kLZ4HCCompression supported: 1 2024-05-03T08:48:50.868 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: kLZ4Compression supported: 1 2024-05-03T08:48:50.868 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: kBZip2Compression supported: 0 2024-05-03T08:48:50.869 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: kZlibCompression supported: 1 2024-05-03T08:48:50.869 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: kSnappyCompression supported: 1 2024-05-03T08:48:50.869 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-03T08:48:50.869 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi032/store.db/MANIFEST-000009 2024-05-03T08:48:50.869 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-05-03T08:48:50.869 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-03T08:48:50.869 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.merge_operator: 2024-05-03T08:48:50.869 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compaction_filter: None 2024-05-03T08:48:50.869 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compaction_filter_factory: None 2024-05-03T08:48:50.869 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.sst_partitioner_factory: None 2024-05-03T08:48:50.869 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-03T08:48:50.869 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-03T08:48:50.869 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557b62b2dd00) 2024-05-03T08:48:50.869 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: cache_index_and_filter_blocks: 1 2024-05-03T08:48:50.869 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-03T08:48:50.870 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-03T08:48:50.870 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: pin_top_level_index_and_filter: 1 2024-05-03T08:48:50.870 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: index_type: 0 2024-05-03T08:48:50.870 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: data_block_index_type: 0 2024-05-03T08:48:50.870 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: index_shortening: 1 2024-05-03T08:48:50.870 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-03T08:48:50.870 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: hash_index_allow_collision: 1 2024-05-03T08:48:50.870 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: checksum: 1 2024-05-03T08:48:50.870 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: no_block_cache: 0 2024-05-03T08:48:50.870 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: block_cache: 0x557b62b98170 2024-05-03T08:48:50.870 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: block_cache_name: BinnedLRUCache 2024-05-03T08:48:50.870 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: block_cache_options: 2024-05-03T08:48:50.870 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: capacity : 536870912 2024-05-03T08:48:50.870 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: num_shard_bits : 4 2024-05-03T08:48:50.871 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: strict_capacity_limit : 0 2024-05-03T08:48:50.871 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: high_pri_pool_ratio: 0.000 2024-05-03T08:48:50.871 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: block_cache_compressed: (nil) 2024-05-03T08:48:50.871 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: persistent_cache: (nil) 2024-05-03T08:48:50.871 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: block_size: 4096 2024-05-03T08:48:50.871 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: block_size_deviation: 10 2024-05-03T08:48:50.871 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: block_restart_interval: 16 2024-05-03T08:48:50.871 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: index_block_restart_interval: 1 2024-05-03T08:48:50.871 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: metadata_block_size: 4096 2024-05-03T08:48:50.871 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: partition_filters: 0 2024-05-03T08:48:50.871 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: use_delta_encoding: 1 2024-05-03T08:48:50.871 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-05-03T08:48:50.871 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: whole_key_filtering: 1 2024-05-03T08:48:50.871 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: verify_compression: 0 2024-05-03T08:48:50.871 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: read_amp_bytes_per_bit: 0 2024-05-03T08:48:50.872 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: format_version: 4 2024-05-03T08:48:50.872 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: enable_index_compression: 1 2024-05-03T08:48:50.872 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: block_align: 0 2024-05-03T08:48:50.872 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.write_buffer_size: 33554432 2024-05-03T08:48:50.872 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_write_buffer_number: 2 2024-05-03T08:48:50.872 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compression: NoCompression 2024-05-03T08:48:50.872 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.bottommost_compression: Disabled 2024-05-03T08:48:50.872 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.prefix_extractor: nullptr 2024-05-03T08:48:50.872 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-03T08:48:50.872 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.num_levels: 7 2024-05-03T08:48:50.872 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-03T08:48:50.872 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-03T08:48:50.872 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-03T08:48:50.872 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-03T08:48:50.872 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-03T08:48:50.872 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-03T08:48:50.873 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-03T08:48:50.873 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-03T08:48:50.873 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-03T08:48:50.873 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-03T08:48:50.873 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-03T08:48:50.873 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compression_opts.level: 32767 2024-05-03T08:48:50.873 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compression_opts.strategy: 0 2024-05-03T08:48:50.873 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-03T08:48:50.873 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-03T08:48:50.873 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-03T08:48:50.873 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compression_opts.enabled: false 2024-05-03T08:48:50.874 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-03T08:48:50.874 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-03T08:48:50.874 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-03T08:48:50.874 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.target_file_size_base: 67108864 2024-05-03T08:48:50.874 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-03T08:48:50.874 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-03T08:48:50.874 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-03T08:48:50.874 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-03T08:48:50.874 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-03T08:48:50.874 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-03T08:48:50.874 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-03T08:48:50.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-03T08:48:50.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-03T08:48:50.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-03T08:48:50.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-03T08:48:50.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-03T08:48:50.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-03T08:48:50.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.arena_block_size: 4194304 2024-05-03T08:48:50.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-03T08:48:50.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-03T08:48:50.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-05-03T08:48:50.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.disable_auto_compactions: 0 2024-05-03T08:48:50.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-03T08:48:50.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-03T08:48:50.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-03T08:48:50.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-03T08:48:50.876 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-03T08:48:50.876 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-03T08:48:50.876 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-03T08:48:50.876 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-03T08:48:50.876 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-03T08:48:50.876 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-03T08:48:50.876 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.table_properties_collectors: 2024-05-03T08:48:50.876 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.inplace_update_support: 0 2024-05-03T08:48:50.876 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-03T08:48:50.876 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-03T08:48:50.876 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-03T08:48:50.876 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-03T08:48:50.876 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.bloom_locality: 0 2024-05-03T08:48:50.876 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.max_successive_merges: 0 2024-05-03T08:48:50.876 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-03T08:48:50.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.paranoid_file_checks: 0 2024-05-03T08:48:50.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.force_consistency_checks: 1 2024-05-03T08:48:50.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.report_bg_io_stats: 0 2024-05-03T08:48:50.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.ttl: 2592000 2024-05-03T08:48:50.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-03T08:48:50.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.enable_blob_files: false 2024-05-03T08:48:50.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.min_blob_size: 0 2024-05-03T08:48:50.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.blob_file_size: 268435456 2024-05-03T08:48:50.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-03T08:48:50.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-03T08:48:50.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-03T08:48:50.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi032/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-03T08:48:50.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-05-03T08:48:50.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-05-03T08:48:50.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714726130688986, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-05-03T08:48:50.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-05-03T08:48:50.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-05-03T08:48:50.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714726130689736, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79387, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77651, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71960, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714726130, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "9c8c60c4-3ecc-47c9-9737-7d9baf3424bd", "db_session_id": "MVZ9TRZ1JGC7LUC3RESX"}} 2024-05-03T08:48:50.878 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-05-03T08:48:50.878 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714726130690083, "job": 1, "event": "recovery_finished"} 2024-05-03T08:48:50.878 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi032/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-03T08:48:50.878 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x557b62b7ea80 2024-05-03T08:48:50.878 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: DB pointer 0x557b62b8e000 2024-05-03T08:48:50.878 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-05-03T08:48:50.878 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-05-03T08:48:50.878 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: ** DB Stats ** 2024-05-03T08:48:50.878 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-03T08:48:50.878 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-03T08:48:50.878 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-03T08:48:50.878 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-03T08:48:50.878 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-03T08:48:50.878 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-05-03T08:48:50.878 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-03T08:48:50.879 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: 2024-05-03T08:48:50.879 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: ** Compaction Stats [default] ** 2024-05-03T08:48:50.879 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-03T08:48:50.879 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-03T08:48:50.879 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: L0 2/0 79.32 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 155.8 0.00 0.00 1 0.000 0 0 2024-05-03T08:48:50.879 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Sum 2/0 79.32 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 155.8 0.00 0.00 1 0.000 0 0 2024-05-03T08:48:50.879 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 155.8 0.00 0.00 1 0.000 0 0 2024-05-03T08:48:50.879 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: 2024-05-03T08:48:50.879 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: ** Compaction Stats [default] ** 2024-05-03T08:48:50.879 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-03T08:48:50.879 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-03T08:48:50.879 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 155.8 0.00 0.00 1 0.000 0 0 2024-05-03T08:48:50.879 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-03T08:48:50.879 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-03T08:48:50.879 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-03T08:48:50.879 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-03T08:48:50.879 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-03T08:48:50.880 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-03T08:48:50.880 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Cumulative compaction: 0.00 GB write, 24.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-03T08:48:50.880 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Interval compaction: 0.00 GB write, 24.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-03T08:48:50.880 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-03T08:48:50.880 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: 2024-05-03T08:48:50.880 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-03T08:48:50.880 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: 2024-05-03T08:48:50.880 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: ** Compaction Stats [default] ** 2024-05-03T08:48:50.880 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-03T08:48:50.880 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-03T08:48:50.880 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: L0 2/0 79.32 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 155.8 0.00 0.00 1 0.000 0 0 2024-05-03T08:48:50.880 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Sum 2/0 79.32 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 155.8 0.00 0.00 1 0.000 0 0 2024-05-03T08:48:50.880 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-05-03T08:48:50.880 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: 2024-05-03T08:48:50.880 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: ** Compaction Stats [default] ** 2024-05-03T08:48:50.880 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-03T08:48:50.881 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-03T08:48:50.881 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 155.8 0.00 0.00 1 0.000 0 0 2024-05-03T08:48:50.881 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-03T08:48:50.881 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-03T08:48:50.881 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-03T08:48:50.881 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-03T08:48:50.881 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-03T08:48:50.881 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-03T08:48:50.881 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Cumulative compaction: 0.00 GB write, 24.27 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-03T08:48:50.881 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-03T08:48:50.881 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-03T08:48:50.881 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: 2024-05-03T08:48:50.881 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-03T08:48:50.881 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: starting mon.smithi032 rank 0 at public addrs [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] at bind addrs [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi032 fsid df63fec2-0929-11ef-bc97-c7b262605968 2024-05-03T08:48:50.882 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: mon.smithi032@-1(???) e1 preinit fsid df63fec2-0929-11ef-bc97-c7b262605968 2024-05-03T08:48:50.882 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: mon.smithi032@-1(???).mds e1 new map 2024-05-03T08:48:50.882 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: mon.smithi032@-1(???).mds e1 print_map 2024-05-03T08:48:50.882 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: e1 2024-05-03T08:48:50.882 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-03T08:48:50.882 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-03T08:48:50.882 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: legacy client fscid: -1 2024-05-03T08:48:50.882 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: 2024-05-03T08:48:50.882 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout: No filesystems configured 2024-05-03T08:48:50.882 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: mon.smithi032@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-05-03T08:48:50.882 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: mon.smithi032@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-03T08:48:50.882 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: mon.smithi032@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-03T08:48:50.882 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: mon.smithi032@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-03T08:48:50.883 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: mon.smithi032@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-05-03T08:48:50.883 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 bash[22384]: 7e5da84a501eb3ff7337cde84c7f3667e36b6b4ce6f199f91b9b974421ca3ff8 2024-05-03T08:48:50.883 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 systemd[1]: Started Ceph mon.smithi032 for df63fec2-0929-11ef-bc97-c7b262605968. 2024-05-03T08:48:50.883 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: mon.smithi032 is new leader, mons smithi032 in quorum (ranks 0) 2024-05-03T08:48:50.883 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: monmap e1: 1 mons at {smithi032=[v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0]} 2024-05-03T08:48:50.883 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: fsmap 2024-05-03T08:48:50.883 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: osdmap e1: 0 total, 0 up, 0 in 2024-05-03T08:48:50.883 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:50 smithi032 ceph-mon[22403]: mgrmap e1: no daemons active 2024-05-03T08:48:52.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:48:52 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/3121849521' entity='client.admin' 2024-05-03T08:48:55.903 INFO:teuthology.orchestra.run.smithi032.stderr:Wrote config to /etc/ceph/ceph.conf 2024-05-03T08:48:55.903 INFO:teuthology.orchestra.run.smithi032.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-05-03T08:48:55.904 INFO:teuthology.orchestra.run.smithi032.stderr:Creating mgr... 2024-05-03T08:48:55.904 INFO:teuthology.orchestra.run.smithi032.stderr:Verifying port 9283 ... 2024-05-03T08:48:56.203 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: Failed to reset failed state of unit ceph-df63fec2-0929-11ef-bc97-c7b262605968@mgr.smithi032.umllay.service: Unit ceph-df63fec2-0929-11ef-bc97-c7b262605968@mgr.smithi032.umllay.service not loaded. 2024-05-03T08:48:56.210 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: Created symlink /etc/systemd/system/ceph-df63fec2-0929-11ef-bc97-c7b262605968.target.wants/ceph-df63fec2-0929-11ef-bc97-c7b262605968@mgr.smithi032.umllay.service → /etc/systemd/system/ceph-df63fec2-0929-11ef-bc97-c7b262605968@.service. 2024-05-03T08:49:00.299 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: disabled 2024-05-03T08:49:00.307 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: inactive 2024-05-03T08:49:00.307 INFO:teuthology.orchestra.run.smithi032.stderr:firewalld.service is not enabled 2024-05-03T08:49:00.307 INFO:teuthology.orchestra.run.smithi032.stderr:Not possible to enable service . firewalld.service is not available 2024-05-03T08:49:00.315 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: disabled 2024-05-03T08:49:00.323 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: inactive 2024-05-03T08:49:00.323 INFO:teuthology.orchestra.run.smithi032.stderr:firewalld.service is not enabled 2024-05-03T08:49:00.323 INFO:teuthology.orchestra.run.smithi032.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-05-03T08:49:00.324 INFO:teuthology.orchestra.run.smithi032.stderr:Waiting for mgr to start... 2024-05-03T08:49:00.324 INFO:teuthology.orchestra.run.smithi032.stderr:Waiting for mgr... 2024-05-03T08:49:03.466 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: 2024-05-03T08:49:03.466 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: { 2024-05-03T08:49:03.466 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "fsid": "df63fec2-0929-11ef-bc97-c7b262605968", 2024-05-03T08:49:03.466 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "health": { 2024-05-03T08:49:03.466 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-03T08:49:03.466 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "checks": {}, 2024-05-03T08:49:03.466 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "mutes": [] 2024-05-03T08:49:03.466 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:03.467 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-03T08:49:03.467 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "quorum": [ 2024-05-03T08:49:03.467 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: 0 2024-05-03T08:49:03.467 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: ], 2024-05-03T08:49:03.467 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-03T08:49:03.467 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "smithi032" 2024-05-03T08:49:03.467 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: ], 2024-05-03T08:49:03.467 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-05-03T08:49:03.467 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "monmap": { 2024-05-03T08:49:03.467 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-03T08:49:03.467 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-03T08:49:03.467 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-03T08:49:03.467 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:03.467 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "osdmap": { 2024-05-03T08:49:03.467 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-03T08:49:03.467 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-03T08:49:03.468 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-03T08:49:03.468 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-03T08:49:03.468 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-03T08:49:03.468 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-03T08:49:03.468 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-03T08:49:03.468 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:03.468 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "pgmap": { 2024-05-03T08:49:03.468 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-03T08:49:03.468 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-03T08:49:03.468 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-03T08:49:03.468 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-03T08:49:03.468 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-03T08:49:03.468 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-03T08:49:03.468 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-03T08:49:03.469 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-03T08:49:03.469 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:03.469 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "fsmap": { 2024-05-03T08:49:03.469 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-03T08:49:03.469 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-03T08:49:03.469 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-03T08:49:03.469 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:03.469 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-03T08:49:03.469 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "available": false, 2024-05-03T08:49:03.469 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-03T08:49:03.469 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "modules": [ 2024-05-03T08:49:03.469 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "iostat", 2024-05-03T08:49:03.469 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "nfs", 2024-05-03T08:49:03.469 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "restful" 2024-05-03T08:49:03.469 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: ], 2024-05-03T08:49:03.469 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "services": {} 2024-05-03T08:49:03.470 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:03.470 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "servicemap": { 2024-05-03T08:49:03.470 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-03T08:49:03.470 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "modified": "2024-05-03T08:48:46.312766+0000", 2024-05-03T08:49:03.470 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "services": {} 2024-05-03T08:49:03.470 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:03.470 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-03T08:49:03.470 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: } 2024-05-03T08:49:03.637 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:03 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/568801496' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-03T08:49:04.092 INFO:teuthology.orchestra.run.smithi032.stderr:mgr not available, waiting (1/15)... 2024-05-03T08:49:06.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:05 smithi032 ceph-mon[22403]: Activating manager daemon smithi032.umllay 2024-05-03T08:49:06.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:05 smithi032 ceph-mon[22403]: mgrmap e2: smithi032.umllay(active, starting, since 0.00270247s) 2024-05-03T08:49:06.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:05 smithi032 ceph-mon[22403]: from='mgr.14100 172.21.15.32:0/2176691356' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-03T08:49:06.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:05 smithi032 ceph-mon[22403]: from='mgr.14100 172.21.15.32:0/2176691356' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-03T08:49:06.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:05 smithi032 ceph-mon[22403]: from='mgr.14100 172.21.15.32:0/2176691356' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-03T08:49:06.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:05 smithi032 ceph-mon[22403]: from='mgr.14100 172.21.15.32:0/2176691356' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi032"}]: dispatch 2024-05-03T08:49:06.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:05 smithi032 ceph-mon[22403]: from='mgr.14100 172.21.15.32:0/2176691356' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mgr metadata", "who": "smithi032.umllay", "id": "smithi032.umllay"}]: dispatch 2024-05-03T08:49:06.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:05 smithi032 ceph-mon[22403]: Manager daemon smithi032.umllay is now available 2024-05-03T08:49:06.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:05 smithi032 ceph-mon[22403]: from='mgr.14100 172.21.15.32:0/2176691356' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:49:06.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:05 smithi032 ceph-mon[22403]: from='mgr.14100 172.21.15.32:0/2176691356' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:49:06.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:05 smithi032 ceph-mon[22403]: from='mgr.14100 172.21.15.32:0/2176691356' entity='mgr.smithi032.umllay' 2024-05-03T08:49:06.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:05 smithi032 ceph-mon[22403]: from='mgr.14100 172.21.15.32:0/2176691356' entity='mgr.smithi032.umllay' 2024-05-03T08:49:06.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:05 smithi032 ceph-mon[22403]: from='mgr.14100 172.21.15.32:0/2176691356' entity='mgr.smithi032.umllay' 2024-05-03T08:49:06.657 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: 2024-05-03T08:49:06.657 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: { 2024-05-03T08:49:06.657 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "fsid": "df63fec2-0929-11ef-bc97-c7b262605968", 2024-05-03T08:49:06.657 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "health": { 2024-05-03T08:49:06.657 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-03T08:49:06.657 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "checks": {}, 2024-05-03T08:49:06.657 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "mutes": [] 2024-05-03T08:49:06.657 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:06.658 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-03T08:49:06.658 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "quorum": [ 2024-05-03T08:49:06.658 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: 0 2024-05-03T08:49:06.658 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: ], 2024-05-03T08:49:06.658 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-03T08:49:06.658 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "smithi032" 2024-05-03T08:49:06.658 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: ], 2024-05-03T08:49:06.658 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "quorum_age": 15, 2024-05-03T08:49:06.658 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "monmap": { 2024-05-03T08:49:06.658 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-03T08:49:06.658 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-03T08:49:06.658 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-03T08:49:06.658 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:06.658 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "osdmap": { 2024-05-03T08:49:06.659 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-03T08:49:06.659 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-03T08:49:06.659 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-03T08:49:06.659 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-03T08:49:06.659 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-03T08:49:06.662 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-03T08:49:06.662 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-03T08:49:06.662 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:06.662 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "pgmap": { 2024-05-03T08:49:06.662 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-03T08:49:06.662 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-03T08:49:06.663 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-03T08:49:06.663 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-03T08:49:06.663 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-03T08:49:06.663 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-03T08:49:06.663 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-03T08:49:06.663 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-03T08:49:06.663 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:06.663 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "fsmap": { 2024-05-03T08:49:06.663 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-03T08:49:06.663 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-03T08:49:06.663 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-03T08:49:06.663 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:06.663 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-03T08:49:06.663 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "available": false, 2024-05-03T08:49:06.663 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-03T08:49:06.664 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "modules": [ 2024-05-03T08:49:06.664 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "iostat", 2024-05-03T08:49:06.664 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "nfs", 2024-05-03T08:49:06.664 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "restful" 2024-05-03T08:49:06.664 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: ], 2024-05-03T08:49:06.664 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "services": {} 2024-05-03T08:49:06.664 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:06.664 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "servicemap": { 2024-05-03T08:49:06.664 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-03T08:49:06.664 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "modified": "2024-05-03T08:48:46.312766+0000", 2024-05-03T08:49:06.664 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "services": {} 2024-05-03T08:49:06.664 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:06.664 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-03T08:49:06.664 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: } 2024-05-03T08:49:06.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:06 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/150883352' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-03T08:49:06.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:06 smithi032 ceph-mon[22403]: mgrmap e3: smithi032.umllay(active, since 1.00627s) 2024-05-03T08:49:07.182 INFO:teuthology.orchestra.run.smithi032.stderr:mgr not available, waiting (2/15)... 2024-05-03T08:49:09.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:08 smithi032 ceph-mon[22403]: mgrmap e4: smithi032.umllay(active, since 2s) 2024-05-03T08:49:09.891 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: 2024-05-03T08:49:09.891 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: { 2024-05-03T08:49:09.891 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "fsid": "df63fec2-0929-11ef-bc97-c7b262605968", 2024-05-03T08:49:09.891 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "health": { 2024-05-03T08:49:09.891 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-03T08:49:09.891 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "checks": {}, 2024-05-03T08:49:09.891 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "mutes": [] 2024-05-03T08:49:09.891 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:09.891 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-03T08:49:09.891 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "quorum": [ 2024-05-03T08:49:09.891 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: 0 2024-05-03T08:49:09.892 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: ], 2024-05-03T08:49:09.892 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-03T08:49:09.892 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "smithi032" 2024-05-03T08:49:09.892 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: ], 2024-05-03T08:49:09.892 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "quorum_age": 19, 2024-05-03T08:49:09.892 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "monmap": { 2024-05-03T08:49:09.892 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-03T08:49:09.892 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-03T08:49:09.892 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-03T08:49:09.892 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:09.892 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "osdmap": { 2024-05-03T08:49:09.892 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-03T08:49:09.892 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-03T08:49:09.892 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-03T08:49:09.892 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-03T08:49:09.892 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-03T08:49:09.893 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-03T08:49:09.893 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-03T08:49:09.893 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:09.893 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "pgmap": { 2024-05-03T08:49:09.893 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-03T08:49:09.893 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-03T08:49:09.893 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-03T08:49:09.893 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-03T08:49:09.893 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-03T08:49:09.893 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-03T08:49:09.893 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-03T08:49:09.893 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-03T08:49:09.893 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:09.893 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "fsmap": { 2024-05-03T08:49:09.893 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-03T08:49:09.894 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-03T08:49:09.894 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-03T08:49:09.894 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:09.894 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-03T08:49:09.894 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "available": true, 2024-05-03T08:49:09.894 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-03T08:49:09.894 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "modules": [ 2024-05-03T08:49:09.894 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "iostat", 2024-05-03T08:49:09.894 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "nfs", 2024-05-03T08:49:09.894 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "restful" 2024-05-03T08:49:09.894 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: ], 2024-05-03T08:49:09.894 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "services": {} 2024-05-03T08:49:09.894 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:09.894 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "servicemap": { 2024-05-03T08:49:09.894 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-03T08:49:09.895 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "modified": "2024-05-03T08:48:46.312766+0000", 2024-05-03T08:49:09.895 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "services": {} 2024-05-03T08:49:09.895 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: }, 2024-05-03T08:49:09.895 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-03T08:49:09.895 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: } 2024-05-03T08:49:10.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:09 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/461937010' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-03T08:49:10.434 INFO:teuthology.orchestra.run.smithi032.stderr:mgr is available 2024-05-03T08:49:11.043 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: 2024-05-03T08:49:11.044 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: [global] 2024-05-03T08:49:11.044 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: fsid = df63fec2-0929-11ef-bc97-c7b262605968 2024-05-03T08:49:11.044 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-05-03T08:49:11.044 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-05-03T08:49:11.044 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-05-03T08:49:11.044 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-05-03T08:49:11.044 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: 2024-05-03T08:49:11.044 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: [mgr] 2024-05-03T08:49:11.044 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-05-03T08:49:11.044 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: 2024-05-03T08:49:11.044 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: [osd] 2024-05-03T08:49:11.044 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-05-03T08:49:11.044 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-05-03T08:49:11.565 INFO:teuthology.orchestra.run.smithi032.stderr:Enabling cephadm module... 2024-05-03T08:49:11.981 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:11 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/1359488194' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-05-03T08:49:13.013 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:12 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/649438907' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-05-03T08:49:13.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:13 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/649438907' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-05-03T08:49:13.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:13 smithi032 ceph-mon[22403]: mgrmap e5: smithi032.umllay(active, since 7s) 2024-05-03T08:49:14.027 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: { 2024-05-03T08:49:14.027 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "epoch": 5, 2024-05-03T08:49:14.027 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "available": true, 2024-05-03T08:49:14.027 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "active_name": "smithi032.umllay", 2024-05-03T08:49:14.027 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_standby": 0 2024-05-03T08:49:14.027 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: } 2024-05-03T08:49:14.751 INFO:teuthology.orchestra.run.smithi032.stderr:Waiting for the mgr to restart... 2024-05-03T08:49:14.751 INFO:teuthology.orchestra.run.smithi032.stderr:Waiting for mgr epoch 5... 2024-05-03T08:49:14.968 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:14 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/2290400928' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-05-03T08:49:20.037 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: { 2024-05-03T08:49:20.037 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-05-03T08:49:20.037 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "initialized": true 2024-05-03T08:49:20.037 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: } 2024-05-03T08:49:20.287 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:19 smithi032 ceph-mon[22403]: Active manager daemon smithi032.umllay restarted 2024-05-03T08:49:20.288 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:19 smithi032 ceph-mon[22403]: Activating manager daemon smithi032.umllay 2024-05-03T08:49:20.288 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:19 smithi032 ceph-mon[22403]: osdmap e2: 0 total, 0 up, 0 in 2024-05-03T08:49:20.288 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:19 smithi032 ceph-mon[22403]: mgrmap e6: smithi032.umllay(active, starting, since 0.0526592s) 2024-05-03T08:49:20.288 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:19 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi032"}]: dispatch 2024-05-03T08:49:20.288 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:19 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mgr metadata", "who": "smithi032.umllay", "id": "smithi032.umllay"}]: dispatch 2024-05-03T08:49:20.288 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:19 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-03T08:49:20.288 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:19 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-03T08:49:20.288 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:19 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-03T08:49:20.288 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:19 smithi032 ceph-mon[22403]: Manager daemon smithi032.umllay is now available 2024-05-03T08:49:20.288 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:19 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' 2024-05-03T08:49:20.288 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:19 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' 2024-05-03T08:49:20.288 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:19 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:49:20.288 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:19 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:49:20.289 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:19 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:49:20.289 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:19 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:49:20.289 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:19 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:49:20.566 INFO:teuthology.orchestra.run.smithi032.stderr:mgr epoch 5 is available 2024-05-03T08:49:20.567 INFO:teuthology.orchestra.run.smithi032.stderr:Setting orchestrator backend to cephadm... 2024-05-03T08:49:21.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:21 smithi032 ceph-mon[22403]: mgrmap e7: smithi032.umllay(active, since 1.05552s) 2024-05-03T08:49:21.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:21 smithi032 ceph-mon[22403]: [03/May/2024:08:49:20] ENGINE Bus STARTING 2024-05-03T08:49:21.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:21 smithi032 ceph-mon[22403]: [03/May/2024:08:49:20] ENGINE Serving on https://172.21.15.32:7150 2024-05-03T08:49:21.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:21 smithi032 ceph-mon[22403]: [03/May/2024:08:49:20] ENGINE Bus STARTED 2024-05-03T08:49:21.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:21 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' 2024-05-03T08:49:21.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:21 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:49:22.386 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:22 smithi032 ceph-mon[22403]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:49:22.387 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:22 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' 2024-05-03T08:49:22.387 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:22 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:49:22.387 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:22 smithi032 ceph-mon[22403]: mgrmap e8: smithi032.umllay(active, since 2s) 2024-05-03T08:49:22.387 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: value unchanged 2024-05-03T08:49:22.885 INFO:teuthology.orchestra.run.smithi032.stderr:Generating ssh key... 2024-05-03T08:49:23.320 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:23 smithi032 ceph-mon[22403]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:49:24.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:24 smithi032 ceph-mon[22403]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:49:24.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:24 smithi032 ceph-mon[22403]: Generating ssh key... 2024-05-03T08:49:24.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:24 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' 2024-05-03T08:49:24.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:24 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' 2024-05-03T08:49:24.910 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCc8ae9lq7j/+ESY5cUVQHxbS5gKe8rVoWQw5ZTFfMq29lTvOP4pyJT9/wGnF2bN4wk7YwZuDJPBY8jUNKCfNnKPVvf9lRUlVgGUVxSZvzaMqf9othtk9th2lvBee/SCNqIcvz+e5sJA3Gem+ydChoY27g1Q2YUYnhgAt1Yi/LhFU9AerqIu3V4YFYPRNYoMYDQmiThY3THlOvK6OQ+3rEcBF/ncxtOWZ1K1v+AsQXXqvxJjs9WcoarorYKp40609fXE/rWNMpg0QrXXK4Zwc8U1IXmKns+EhbexheOOHeaduDeGRaOz8iIajPhsoBcR4CkPRPNQEFpXJOeRkQHvIp+GpffHd3TPeAKpFMhofqmFVaeA59SpKf8WeG+GA/AZ1PltVDqFFBNFTshflS6kyR1++ycGsjbDkYtUF6O5idUrRicC+p/dIZvgAFw6lcEKsn7PpAuZkn2fAWaV8hw7FE5xgHRGDz95sPIPJFZkJ5264T1C2p7DJ1gt736w2wolLE= ceph-df63fec2-0929-11ef-bc97-c7b262605968 2024-05-03T08:49:25.550 INFO:teuthology.orchestra.run.smithi032.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-05-03T08:49:25.550 INFO:teuthology.orchestra.run.smithi032.stderr:Adding key to root@localhost authorized_keys... 2024-05-03T08:49:25.550 INFO:teuthology.orchestra.run.smithi032.stderr:Adding host smithi032... 2024-05-03T08:49:27.034 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:26 smithi032 ceph-mon[22403]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:49:27.041 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: Added host 'smithi032' with addr '172.21.15.32' 2024-05-03T08:49:27.589 INFO:teuthology.orchestra.run.smithi032.stderr:Deploying mon service with default placement... 2024-05-03T08:49:27.840 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:27 smithi032 ceph-mon[22403]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi032", "addr": "172.21.15.32", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:49:27.840 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:27 smithi032 ceph-mon[22403]: Deploying cephadm binary to smithi032 2024-05-03T08:49:27.840 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:27 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' 2024-05-03T08:49:27.840 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:27 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:49:28.221 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: Scheduled mon update... 2024-05-03T08:49:28.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:28 smithi032 ceph-mon[22403]: Added host smithi032 2024-05-03T08:49:28.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:28 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' 2024-05-03T08:49:29.047 INFO:teuthology.orchestra.run.smithi032.stderr:Deploying mgr service with default placement... 2024-05-03T08:49:29.679 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-05-03T08:49:29.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:29 smithi032 ceph-mon[22403]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:49:29.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:29 smithi032 ceph-mon[22403]: Saving service mon spec with placement count:5 2024-05-03T08:49:29.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:29 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' 2024-05-03T08:49:31.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:30 smithi032 ceph-mon[22403]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:49:31.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:30 smithi032 ceph-mon[22403]: Saving service mgr spec with placement count:2 2024-05-03T08:49:31.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:30 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' 2024-05-03T08:49:31.143 INFO:teuthology.orchestra.run.smithi032.stderr:Deploying crash service with default placement... 2024-05-03T08:49:31.824 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: Scheduled crash update... 2024-05-03T08:49:32.374 INFO:teuthology.orchestra.run.smithi032.stderr:Deploying prometheus service with default placement... 2024-05-03T08:49:32.498 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:32 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' 2024-05-03T08:49:32.498 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:32 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' 2024-05-03T08:49:33.328 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:33 smithi032 ceph-mon[22403]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:49:33.329 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:33 smithi032 ceph-mon[22403]: Saving service crash spec with placement * 2024-05-03T08:49:33.329 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:33 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' 2024-05-03T08:49:33.329 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-05-03T08:49:33.979 INFO:teuthology.orchestra.run.smithi032.stderr:Deploying grafana service with default placement... 2024-05-03T08:49:34.570 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:34 smithi032 ceph-mon[22403]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:49:34.570 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:34 smithi032 ceph-mon[22403]: Saving service prometheus spec with placement count:1 2024-05-03T08:49:34.570 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:34 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' 2024-05-03T08:49:34.570 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:34 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' 2024-05-03T08:49:34.605 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-05-03T08:49:35.625 INFO:teuthology.orchestra.run.smithi032.stderr:Deploying node-exporter service with default placement... 2024-05-03T08:49:35.832 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:35 smithi032 ceph-mon[22403]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:49:35.832 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:35 smithi032 ceph-mon[22403]: Saving service grafana spec with placement count:1 2024-05-03T08:49:35.832 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:35 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' 2024-05-03T08:49:36.659 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-05-03T08:49:37.277 INFO:teuthology.orchestra.run.smithi032.stderr:Deploying alertmanager service with default placement... 2024-05-03T08:49:37.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:37 smithi032 ceph-mon[22403]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:49:37.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:37 smithi032 ceph-mon[22403]: Saving service node-exporter spec with placement * 2024-05-03T08:49:37.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:37 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' 2024-05-03T08:49:37.939 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-05-03T08:49:39.191 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:38 smithi032 ceph-mon[22403]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:49:39.192 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:38 smithi032 ceph-mon[22403]: Saving service alertmanager spec with placement count:1 2024-05-03T08:49:39.192 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:38 smithi032 ceph-mon[22403]: from='mgr.14118 172.21.15.32:0/2420513720' entity='mgr.smithi032.umllay' 2024-05-03T08:49:40.469 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:40 smithi032 ceph-mon[22403]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:49:40.470 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:40 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/2455582877' entity='client.admin' 2024-05-03T08:49:41.100 INFO:teuthology.orchestra.run.smithi032.stderr:Enabling the dashboard module... 2024-05-03T08:49:41.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:41 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/114624033' entity='client.admin' 2024-05-03T08:49:42.992 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:42 smithi032 ceph-mon[22403]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:49:42.992 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:42 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/1581452634' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-05-03T08:49:43.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:43 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/1581452634' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-05-03T08:49:43.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:43 smithi032 ceph-mon[22403]: mgrmap e9: smithi032.umllay(active, since 23s) 2024-05-03T08:49:44.083 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: { 2024-05-03T08:49:44.083 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "epoch": 9, 2024-05-03T08:49:44.083 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "available": true, 2024-05-03T08:49:44.083 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "active_name": "smithi032.umllay", 2024-05-03T08:49:44.083 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "num_standby": 0 2024-05-03T08:49:44.083 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: } 2024-05-03T08:49:44.602 INFO:teuthology.orchestra.run.smithi032.stderr:Waiting for the mgr to restart... 2024-05-03T08:49:44.603 INFO:teuthology.orchestra.run.smithi032.stderr:Waiting for mgr epoch 9... 2024-05-03T08:49:44.856 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:44 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/2681564523' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-05-03T08:49:50.076 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: { 2024-05-03T08:49:50.076 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-05-03T08:49:50.076 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: "initialized": true 2024-05-03T08:49:50.077 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: } 2024-05-03T08:49:50.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:50 smithi032 ceph-mon[22403]: Active manager daemon smithi032.umllay restarted 2024-05-03T08:49:50.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:50 smithi032 ceph-mon[22403]: Activating manager daemon smithi032.umllay 2024-05-03T08:49:50.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:50 smithi032 ceph-mon[22403]: osdmap e3: 0 total, 0 up, 0 in 2024-05-03T08:49:50.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:50 smithi032 ceph-mon[22403]: mgrmap e10: smithi032.umllay(active, starting, since 0.0527147s) 2024-05-03T08:49:50.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:50 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi032"}]: dispatch 2024-05-03T08:49:50.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:50 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mgr metadata", "who": "smithi032.umllay", "id": "smithi032.umllay"}]: dispatch 2024-05-03T08:49:50.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:50 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-03T08:49:50.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:50 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-03T08:49:50.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:50 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-03T08:49:50.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:50 smithi032 ceph-mon[22403]: Manager daemon smithi032.umllay is now available 2024-05-03T08:49:50.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:50 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:49:50.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:50 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:49:50.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:50 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:49:50.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:50 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:49:50.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:50 smithi032 ceph-mon[22403]: [03/May/2024:08:49:49] ENGINE Bus STARTING 2024-05-03T08:49:50.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:50 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:49:50.757 INFO:teuthology.orchestra.run.smithi032.stderr:mgr epoch 9 is available 2024-05-03T08:49:50.757 INFO:teuthology.orchestra.run.smithi032.stderr:Generating a dashboard self-signed certificate... 2024-05-03T08:49:51.262 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:51 smithi032 ceph-mon[22403]: [03/May/2024:08:49:49] ENGINE Serving on https://172.21.15.32:7150 2024-05-03T08:49:51.262 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:51 smithi032 ceph-mon[22403]: [03/May/2024:08:49:49] ENGINE Bus STARTED 2024-05-03T08:49:51.262 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:51 smithi032 ceph-mon[22403]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-05-03T08:49:51.262 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:51 smithi032 ceph-mon[22403]: mgrmap e11: smithi032.umllay(active, since 1.05523s) 2024-05-03T08:49:51.262 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:51 smithi032 ceph-mon[22403]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-05-03T08:49:51.517 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: Self-signed certificate created 2024-05-03T08:49:52.197 INFO:teuthology.orchestra.run.smithi032.stderr:Creating initial admin user... 2024-05-03T08:49:52.469 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:52 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:49:52.470 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:52 smithi032 ceph-mon[22403]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:49:52.470 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:52 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:49:52.470 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:52 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:49:52.470 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:52 smithi032 ceph-mon[22403]: mgrmap e12: smithi032.umllay(active, since 2s) 2024-05-03T08:49:52.470 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:52 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:49:53.173 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$DydTzVNVfyc8.x9SU3bxV.Gnj0EVNsd7vKJylIJFkYofJWruJiQKC", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1714726193, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-05-03T08:49:54.561 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:54 smithi032 ceph-mon[22403]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:49:54.561 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:54 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:49:54.563 INFO:teuthology.orchestra.run.smithi032.stderr:Fetching dashboard port number... 2024-05-03T08:49:55.185 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: 8443 2024-05-03T08:49:55.967 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: disabled 2024-05-03T08:49:55.975 INFO:teuthology.orchestra.run.smithi032.stderr:systemctl: inactive 2024-05-03T08:49:55.976 INFO:teuthology.orchestra.run.smithi032.stderr:firewalld.service is not enabled 2024-05-03T08:49:55.976 INFO:teuthology.orchestra.run.smithi032.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-05-03T08:49:55.978 INFO:teuthology.orchestra.run.smithi032.stderr:Ceph Dashboard is now available at: 2024-05-03T08:49:55.979 INFO:teuthology.orchestra.run.smithi032.stderr: 2024-05-03T08:49:55.979 INFO:teuthology.orchestra.run.smithi032.stderr: URL: https://smithi032:8443/ 2024-05-03T08:49:55.979 INFO:teuthology.orchestra.run.smithi032.stderr: User: admin 2024-05-03T08:49:55.979 INFO:teuthology.orchestra.run.smithi032.stderr: Password: mn43qcoj56 2024-05-03T08:49:55.979 INFO:teuthology.orchestra.run.smithi032.stderr: 2024-05-03T08:49:55.979 INFO:teuthology.orchestra.run.smithi032.stderr:Enabling autotune for osd_memory_target 2024-05-03T08:49:56.224 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:56 smithi032 ceph-mon[22403]: mgrmap e13: smithi032.umllay(active, since 6s) 2024-05-03T08:49:56.224 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:56 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/126446065' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-05-03T08:49:58.025 INFO:teuthology.orchestra.run.smithi032.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-05-03T08:49:58.634 INFO:teuthology.orchestra.run.smithi032.stderr:You can access the Ceph CLI with: 2024-05-03T08:49:58.635 INFO:teuthology.orchestra.run.smithi032.stderr: 2024-05-03T08:49:58.635 INFO:teuthology.orchestra.run.smithi032.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-05-03T08:49:58.635 INFO:teuthology.orchestra.run.smithi032.stderr: 2024-05-03T08:49:58.635 INFO:teuthology.orchestra.run.smithi032.stderr:Please consider enabling telemetry to help improve Ceph: 2024-05-03T08:49:58.635 INFO:teuthology.orchestra.run.smithi032.stderr: 2024-05-03T08:49:58.635 INFO:teuthology.orchestra.run.smithi032.stderr: ceph telemetry on 2024-05-03T08:49:58.635 INFO:teuthology.orchestra.run.smithi032.stderr: 2024-05-03T08:49:58.635 INFO:teuthology.orchestra.run.smithi032.stderr:For more information see: 2024-05-03T08:49:58.635 INFO:teuthology.orchestra.run.smithi032.stderr: 2024-05-03T08:49:58.635 INFO:teuthology.orchestra.run.smithi032.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-05-03T08:49:58.635 INFO:teuthology.orchestra.run.smithi032.stderr: 2024-05-03T08:49:58.635 INFO:teuthology.orchestra.run.smithi032.stderr:Bootstrap complete. 2024-05-03T08:49:58.676 INFO:tasks.cephadm:Fetching config... 2024-05-03T08:49:58.677 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-05-03T08:49:58.677 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-05-03T08:49:58.695 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-05-03T08:49:58.695 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-05-03T08:49:58.695 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-05-03T08:49:58.752 INFO:tasks.cephadm:Fetching mon keyring... 2024-05-03T08:49:58.752 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-05-03T08:49:58.752 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd if=/var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/keyring of=/dev/stdout 2024-05-03T08:49:58.820 INFO:tasks.cephadm:Fetching pub ssh key... 2024-05-03T08:49:58.820 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-05-03T08:49:58.820 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-05-03T08:49:58.878 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-05-03T08:49:58.878 DEBUG:teuthology.orchestra.run.smithi032:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCc8ae9lq7j/+ESY5cUVQHxbS5gKe8rVoWQw5ZTFfMq29lTvOP4pyJT9/wGnF2bN4wk7YwZuDJPBY8jUNKCfNnKPVvf9lRUlVgGUVxSZvzaMqf9othtk9th2lvBee/SCNqIcvz+e5sJA3Gem+ydChoY27g1Q2YUYnhgAt1Yi/LhFU9AerqIu3V4YFYPRNYoMYDQmiThY3THlOvK6OQ+3rEcBF/ncxtOWZ1K1v+AsQXXqvxJjs9WcoarorYKp40609fXE/rWNMpg0QrXXK4Zwc8U1IXmKns+EhbexheOOHeaduDeGRaOz8iIajPhsoBcR4CkPRPNQEFpXJOeRkQHvIp+GpffHd3TPeAKpFMhofqmFVaeA59SpKf8WeG+GA/AZ1PltVDqFFBNFTshflS6kyR1++ycGsjbDkYtUF6O5idUrRicC+p/dIZvgAFw6lcEKsn7PpAuZkn2fAWaV8hw7FE5xgHRGDz95sPIPJFZkJ5264T1C2p7DJ1gt736w2wolLE= ceph-df63fec2-0929-11ef-bc97-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-05-03T08:49:58.961 INFO:teuthology.orchestra.run.smithi032.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCc8ae9lq7j/+ESY5cUVQHxbS5gKe8rVoWQw5ZTFfMq29lTvOP4pyJT9/wGnF2bN4wk7YwZuDJPBY8jUNKCfNnKPVvf9lRUlVgGUVxSZvzaMqf9othtk9th2lvBee/SCNqIcvz+e5sJA3Gem+ydChoY27g1Q2YUYnhgAt1Yi/LhFU9AerqIu3V4YFYPRNYoMYDQmiThY3THlOvK6OQ+3rEcBF/ncxtOWZ1K1v+AsQXXqvxJjs9WcoarorYKp40609fXE/rWNMpg0QrXXK4Zwc8U1IXmKns+EhbexheOOHeaduDeGRaOz8iIajPhsoBcR4CkPRPNQEFpXJOeRkQHvIp+GpffHd3TPeAKpFMhofqmFVaeA59SpKf8WeG+GA/AZ1PltVDqFFBNFTshflS6kyR1++ycGsjbDkYtUF6O5idUrRicC+p/dIZvgAFw6lcEKsn7PpAuZkn2fAWaV8hw7FE5xgHRGDz95sPIPJFZkJ5264T1C2p7DJ1gt736w2wolLE= ceph-df63fec2-0929-11ef-bc97-c7b262605968 2024-05-03T08:49:58.973 DEBUG:teuthology.orchestra.run.smithi123:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCc8ae9lq7j/+ESY5cUVQHxbS5gKe8rVoWQw5ZTFfMq29lTvOP4pyJT9/wGnF2bN4wk7YwZuDJPBY8jUNKCfNnKPVvf9lRUlVgGUVxSZvzaMqf9othtk9th2lvBee/SCNqIcvz+e5sJA3Gem+ydChoY27g1Q2YUYnhgAt1Yi/LhFU9AerqIu3V4YFYPRNYoMYDQmiThY3THlOvK6OQ+3rEcBF/ncxtOWZ1K1v+AsQXXqvxJjs9WcoarorYKp40609fXE/rWNMpg0QrXXK4Zwc8U1IXmKns+EhbexheOOHeaduDeGRaOz8iIajPhsoBcR4CkPRPNQEFpXJOeRkQHvIp+GpffHd3TPeAKpFMhofqmFVaeA59SpKf8WeG+GA/AZ1PltVDqFFBNFTshflS6kyR1++ycGsjbDkYtUF6O5idUrRicC+p/dIZvgAFw6lcEKsn7PpAuZkn2fAWaV8hw7FE5xgHRGDz95sPIPJFZkJ5264T1C2p7DJ1gt736w2wolLE= ceph-df63fec2-0929-11ef-bc97-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-05-03T08:49:59.013 INFO:teuthology.orchestra.run.smithi123.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCc8ae9lq7j/+ESY5cUVQHxbS5gKe8rVoWQw5ZTFfMq29lTvOP4pyJT9/wGnF2bN4wk7YwZuDJPBY8jUNKCfNnKPVvf9lRUlVgGUVxSZvzaMqf9othtk9th2lvBee/SCNqIcvz+e5sJA3Gem+ydChoY27g1Q2YUYnhgAt1Yi/LhFU9AerqIu3V4YFYPRNYoMYDQmiThY3THlOvK6OQ+3rEcBF/ncxtOWZ1K1v+AsQXXqvxJjs9WcoarorYKp40609fXE/rWNMpg0QrXXK4Zwc8U1IXmKns+EhbexheOOHeaduDeGRaOz8iIajPhsoBcR4CkPRPNQEFpXJOeRkQHvIp+GpffHd3TPeAKpFMhofqmFVaeA59SpKf8WeG+GA/AZ1PltVDqFFBNFTshflS6kyR1++ycGsjbDkYtUF6O5idUrRicC+p/dIZvgAFw6lcEKsn7PpAuZkn2fAWaV8hw7FE5xgHRGDz95sPIPJFZkJ5264T1C2p7DJ1gt736w2wolLE= ceph-df63fec2-0929-11ef-bc97-c7b262605968 2024-05-03T08:49:59.025 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-05-03T08:49:59.337 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:49:59 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/1396551586' entity='client.admin' 2024-05-03T08:50:01.743 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:01 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:50:01.744 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:01 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd/host:smithi032", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:50:01.744 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:01 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:50:01.744 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:01 smithi032 ceph-mon[22403]: Deploying daemon alertmanager.smithi032 on smithi032 2024-05-03T08:50:02.538 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-05-03T08:50:02.538 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-05-03T08:50:03.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:02 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/3880827390' entity='client.admin' 2024-05-03T08:50:06.413 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi123 2024-05-03T08:50:06.413 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-05-03T08:50:06.413 DEBUG:teuthology.orchestra.run.smithi123:> dd of=/etc/ceph/ceph.conf 2024-05-03T08:50:06.431 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-05-03T08:50:06.431 DEBUG:teuthology.orchestra.run.smithi123:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-03T08:50:06.489 INFO:tasks.cephadm:Adding host smithi123 to orchestrator... 2024-05-03T08:50:06.489 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph orch host add smithi123 2024-05-03T08:50:06.511 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:06 smithi032 ceph-mon[22403]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:50:06.511 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:06 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:50:10.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:10 smithi032 ceph-mon[22403]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:50:12.202 INFO:teuthology.orchestra.run.smithi032.stdout:Added host 'smithi123' with addr '172.21.15.123' 2024-05-03T08:50:12.923 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph orch host ls --format=json 2024-05-03T08:50:12.988 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:12 smithi032 ceph-mon[22403]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:50:12.988 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:12 smithi032 ceph-mon[22403]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi123", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:50:12.989 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:12 smithi032 ceph-mon[22403]: Deploying cephadm binary to smithi123 2024-05-03T08:50:12.989 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:12 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:50:13.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:13 smithi032 ceph-mon[22403]: Added host smithi123 2024-05-03T08:50:13.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:13 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:50:13.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:13 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi032", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-03T08:50:13.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:13 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi032", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-05-03T08:50:13.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:13 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:50:14.960 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:14 smithi032 ceph-mon[22403]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:50:14.960 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:14 smithi032 ceph-mon[22403]: Deploying daemon crash.smithi032 on smithi032 2024-05-03T08:50:14.960 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:14 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:50:15.050 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:50:15.050 INFO:teuthology.orchestra.run.smithi032.stdout:[{"addr": "172.21.15.32", "hostname": "smithi032", "labels": [], "status": ""}, {"addr": "172.21.15.123", "hostname": "smithi123", "labels": [], "status": ""}] 2024-05-03T08:50:16.568 INFO:tasks.cephadm:Setting crush tunables to default 2024-05-03T08:50:16.568 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph osd crush tunables default 2024-05-03T08:50:16.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:16 smithi032 ceph-mon[22403]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-03T08:50:16.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:16 smithi032 ceph-mon[22403]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:50:18.376 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:18 smithi032 ceph-mon[22403]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:50:18.376 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:18 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:50:18.376 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:18 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:50:18.376 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:18 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:50:18.376 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:18 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-03T08:50:18.377 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:18 smithi032 ceph-mon[22403]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-03T08:50:18.377 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:18 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:50:18.377 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:18 smithi032 ceph-mon[22403]: Deploying daemon grafana.smithi032 on smithi032 2024-05-03T08:50:19.157 INFO:teuthology.orchestra.run.smithi032.stderr:adjusted tunables profile to default 2024-05-03T08:50:19.560 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:19 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/2114888214' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-05-03T08:50:19.561 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:19 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:50:20.127 INFO:tasks.cephadm:Adding mon.smithi032 on smithi032 2024-05-03T08:50:20.128 INFO:tasks.cephadm:Adding mon.smithi123 on smithi123 2024-05-03T08:50:20.128 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph orch apply mon '2;smithi032:172.21.15.32=smithi032;smithi123:172.21.15.123=smithi123' 2024-05-03T08:50:20.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:20 smithi032 ceph-mon[22403]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:50:20.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:20 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/2114888214' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-05-03T08:50:20.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:20 smithi032 ceph-mon[22403]: osdmap e4: 0 total, 0 up, 0 in 2024-05-03T08:50:21.963 INFO:teuthology.orchestra.run.smithi123.stdout:Scheduled mon update... 2024-05-03T08:50:22.517 DEBUG:teuthology.orchestra.run.smithi123:mon.smithi123> sudo journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@mon.smithi123.service 2024-05-03T08:50:22.519 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-03T08:50:22.519 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-03T08:50:23.012 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:22 smithi032 ceph-mon[22403]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:50:23.012 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:22 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:50:24.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:23 smithi032 ceph-mon[22403]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi032:172.21.15.32=smithi032;smithi123:172.21.15.123=smithi123", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:50:24.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:23 smithi032 ceph-mon[22403]: Saving service mon spec with placement smithi032:172.21.15.32=smithi032;smithi123:172.21.15.123=smithi123;count:2 2024-05-03T08:50:24.366 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-05-03T08:50:24.366 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":1,"fsid":"df63fec2-0929-11ef-bc97-c7b262605968","modified":"2024-05-03T08:48:43.658624Z","created":"2024-05-03T08:48:43.658624Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi032","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:3300","nonce":0},{"type":"v1","addr":"172.21.15.32:6789","nonce":0}]},"addr":"172.21.15.32:6789/0","public_addr":"172.21.15.32:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-03T08:50:24.368 INFO:teuthology.orchestra.run.smithi123.stderr:dumped monmap epoch 1 2024-05-03T08:50:25.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:24 smithi032 ceph-mon[22403]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:50:25.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:24 smithi032 ceph-mon[22403]: from='client.? 172.21.15.123:0/3285303930' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-03T08:50:25.953 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-03T08:50:25.954 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-03T08:50:27.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:26 smithi032 ceph-mon[22403]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:50:27.785 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-05-03T08:50:27.785 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":1,"fsid":"df63fec2-0929-11ef-bc97-c7b262605968","modified":"2024-05-03T08:48:43.658624Z","created":"2024-05-03T08:48:43.658624Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi032","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:3300","nonce":0},{"type":"v1","addr":"172.21.15.32:6789","nonce":0}]},"addr":"172.21.15.32:6789/0","public_addr":"172.21.15.32:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-03T08:50:27.787 INFO:teuthology.orchestra.run.smithi123.stderr:dumped monmap epoch 1 2024-05-03T08:50:29.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:28 smithi032 ceph-mon[22403]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:50:29.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:28 smithi032 ceph-mon[22403]: from='client.? 172.21.15.123:0/1952803051' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-03T08:50:29.364 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-03T08:50:29.365 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-03T08:50:30.967 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:30 smithi032 ceph-mon[22403]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:50:30.968 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:30 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:50:31.283 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-05-03T08:50:31.284 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":1,"fsid":"df63fec2-0929-11ef-bc97-c7b262605968","modified":"2024-05-03T08:48:43.658624Z","created":"2024-05-03T08:48:43.658624Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi032","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:3300","nonce":0},{"type":"v1","addr":"172.21.15.32:6789","nonce":0}]},"addr":"172.21.15.32:6789/0","public_addr":"172.21.15.32:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-03T08:50:31.285 INFO:teuthology.orchestra.run.smithi123.stderr:dumped monmap epoch 1 2024-05-03T08:50:32.137 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:31 smithi032 ceph-mon[22403]: Deploying daemon node-exporter.smithi032 on smithi032 2024-05-03T08:50:32.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:31 smithi032 ceph-mon[22403]: from='client.? 172.21.15.123:0/2158214803' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-03T08:50:32.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:32 smithi032 ceph-mon[22403]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:50:32.901 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-03T08:50:32.901 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-03T08:50:34.546 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-05-03T08:50:34.546 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":1,"fsid":"df63fec2-0929-11ef-bc97-c7b262605968","modified":"2024-05-03T08:48:43.658624Z","created":"2024-05-03T08:48:43.658624Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi032","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:3300","nonce":0},{"type":"v1","addr":"172.21.15.32:6789","nonce":0}]},"addr":"172.21.15.32:6789/0","public_addr":"172.21.15.32:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-03T08:50:34.547 INFO:teuthology.orchestra.run.smithi123.stderr:dumped monmap epoch 1 2024-05-03T08:50:34.711 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:34 smithi032 ceph-mon[22403]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:50:34.711 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:34 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:50:34.711 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:34 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-05-03T08:50:34.711 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:34 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' 2024-05-03T08:50:35.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:35 smithi032 ceph-mon[22403]: from='mgr.14160 172.21.15.32:0/1828552221' entity='mgr.smithi032.umllay' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-05-03T08:50:35.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:35 smithi032 ceph-mon[22403]: mgrmap e14: smithi032.umllay(active, since 45s) 2024-05-03T08:50:35.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:35 smithi032 ceph-mon[22403]: from='client.? 172.21.15.123:0/1710694183' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-03T08:50:36.079 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-03T08:50:36.079 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-03T08:50:37.942 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-05-03T08:50:37.942 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":1,"fsid":"df63fec2-0929-11ef-bc97-c7b262605968","modified":"2024-05-03T08:48:43.658624Z","created":"2024-05-03T08:48:43.658624Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi032","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:3300","nonce":0},{"type":"v1","addr":"172.21.15.32:6789","nonce":0}]},"addr":"172.21.15.32:6789/0","public_addr":"172.21.15.32:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-03T08:50:37.943 INFO:teuthology.orchestra.run.smithi123.stderr:dumped monmap epoch 1 2024-05-03T08:50:38.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:37 smithi032 ceph-mon[22403]: from='client.? 172.21.15.123:0/4133618650' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-03T08:50:39.499 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-03T08:50:39.499 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-03T08:50:41.386 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-05-03T08:50:41.386 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":1,"fsid":"df63fec2-0929-11ef-bc97-c7b262605968","modified":"2024-05-03T08:48:43.658624Z","created":"2024-05-03T08:48:43.658624Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi032","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:3300","nonce":0},{"type":"v1","addr":"172.21.15.32:6789","nonce":0}]},"addr":"172.21.15.32:6789/0","public_addr":"172.21.15.32:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-03T08:50:41.388 INFO:teuthology.orchestra.run.smithi123.stderr:dumped monmap epoch 1 2024-05-03T08:50:42.003 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:41 smithi032 ceph-mon[22403]: Active manager daemon smithi032.umllay restarted 2024-05-03T08:50:42.003 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:41 smithi032 ceph-mon[22403]: Activating manager daemon smithi032.umllay 2024-05-03T08:50:42.003 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:41 smithi032 ceph-mon[22403]: osdmap e5: 0 total, 0 up, 0 in 2024-05-03T08:50:42.003 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:41 smithi032 ceph-mon[22403]: from='client.? 172.21.15.123:0/3469482483' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-03T08:50:42.004 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:41 smithi032 ceph-mon[22403]: mgrmap e15: smithi032.umllay(active, starting, since 0.974209s) 2024-05-03T08:50:42.004 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:41 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi032"}]: dispatch 2024-05-03T08:50:42.004 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:41 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mgr metadata", "who": "smithi032.umllay", "id": "smithi032.umllay"}]: dispatch 2024-05-03T08:50:42.004 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:41 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-03T08:50:42.004 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:41 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-03T08:50:42.004 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:41 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-03T08:50:42.004 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:41 smithi032 ceph-mon[22403]: Manager daemon smithi032.umllay is now available 2024-05-03T08:50:42.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:50:42.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:50:42.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:50:42.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:50:42.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:50:42.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:50:42.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:50:42.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:42 smithi032 ceph-mon[22403]: [03/May/2024:08:50:42] ENGINE Bus STARTING 2024-05-03T08:50:42.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:42 smithi032 ceph-mon[22403]: [03/May/2024:08:50:42] ENGINE Serving on https://172.21.15.32:7150 2024-05-03T08:50:42.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:42 smithi032 ceph-mon[22403]: [03/May/2024:08:50:42] ENGINE Bus STARTED 2024-05-03T08:50:42.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:50:42.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:50:42.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:50:42.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:42 smithi032 ceph-mon[22403]: mgrmap e16: smithi032.umllay(active, since 1.97602s) 2024-05-03T08:50:42.935 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-03T08:50:42.936 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-03T08:50:44.387 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:44 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:50:44.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:44 smithi032 ceph-mon[22403]: mgrmap e17: smithi032.umllay(active, since 2s) 2024-05-03T08:50:44.768 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-05-03T08:50:44.768 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":1,"fsid":"df63fec2-0929-11ef-bc97-c7b262605968","modified":"2024-05-03T08:48:43.658624Z","created":"2024-05-03T08:48:43.658624Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi032","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:3300","nonce":0},{"type":"v1","addr":"172.21.15.32:6789","nonce":0}]},"addr":"172.21.15.32:6789/0","public_addr":"172.21.15.32:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-03T08:50:44.770 INFO:teuthology.orchestra.run.smithi123.stderr:dumped monmap epoch 1 2024-05-03T08:50:45.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:45 smithi032 ceph-mon[22403]: from='client.? 172.21.15.123:0/1957328368' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-03T08:50:45.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:45 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:50:46.861 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-03T08:50:46.861 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-03T08:50:46.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:46 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:50:46.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:46 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd/host:smithi032", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:50:46.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:46 smithi032 ceph-mon[22403]: Updating smithi032:/etc/ceph/ceph.conf 2024-05-03T08:50:46.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:46 smithi032 ceph-mon[22403]: Updating smithi032:/etc/ceph/ceph.client.admin.keyring 2024-05-03T08:50:46.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:46 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:50:48.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:48 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:50:48.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:48 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:50:48.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:48 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd/host:smithi123", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:50:48.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:48 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:50:48.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:48 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-03T08:50:48.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:48 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-05-03T08:50:48.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:48 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:50:48.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:48 smithi032 ceph-mon[22403]: Deploying daemon crash.smithi123 on smithi123 2024-05-03T08:50:50.146 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-05-03T08:50:50.146 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":1,"fsid":"df63fec2-0929-11ef-bc97-c7b262605968","modified":"2024-05-03T08:48:43.658624Z","created":"2024-05-03T08:48:43.658624Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi032","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:3300","nonce":0},{"type":"v1","addr":"172.21.15.32:6789","nonce":0}]},"addr":"172.21.15.32:6789/0","public_addr":"172.21.15.32:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-03T08:50:50.147 INFO:teuthology.orchestra.run.smithi123.stderr:dumped monmap epoch 1 2024-05-03T08:50:50.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:50 smithi032 ceph-mon[22403]: from='client.? 172.21.15.123:0/2145028177' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-03T08:50:51.715 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-03T08:50:51.715 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-03T08:50:52.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:52 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:50:52.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:52 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi123.xrxinh", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-03T08:50:52.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:52 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi123.xrxinh", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-03T08:50:52.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:52 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-03T08:50:52.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:52 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:50:52.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:52 smithi032 ceph-mon[22403]: Deploying daemon mgr.smithi123.xrxinh on smithi123 2024-05-03T08:50:52.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:52 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:50:54.179 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-05-03T08:50:54.179 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":1,"fsid":"df63fec2-0929-11ef-bc97-c7b262605968","modified":"2024-05-03T08:48:43.658624Z","created":"2024-05-03T08:48:43.658624Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi032","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:3300","nonce":0},{"type":"v1","addr":"172.21.15.32:6789","nonce":0}]},"addr":"172.21.15.32:6789/0","public_addr":"172.21.15.32:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-03T08:50:54.189 INFO:teuthology.orchestra.run.smithi123.stderr:dumped monmap epoch 1 2024-05-03T08:50:54.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:54 smithi032 ceph-mon[22403]: from='client.? 172.21.15.123:0/5672485' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-03T08:50:55.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:55 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:50:55.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:55 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-03T08:50:55.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:55 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:50:56.030 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-03T08:50:56.030 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-03T08:50:56.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:56 smithi032 ceph-mon[22403]: Deploying daemon mon.smithi123 on smithi123 2024-05-03T08:50:58.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:57 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:50:58.809 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-05-03T08:50:58.809 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":1,"fsid":"df63fec2-0929-11ef-bc97-c7b262605968","modified":"2024-05-03T08:48:43.658624Z","created":"2024-05-03T08:48:43.658624Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi032","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:3300","nonce":0},{"type":"v1","addr":"172.21.15.32:6789","nonce":0}]},"addr":"172.21.15.32:6789/0","public_addr":"172.21.15.32:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-03T08:50:58.811 INFO:teuthology.orchestra.run.smithi123.stderr:dumped monmap epoch 1 2024-05-03T08:50:59.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:50:58 smithi032 ceph-mon[22403]: from='client.? 172.21.15.123:0/688400420' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-03T08:51:00.699 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-03T08:51:00.700 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-03T08:51:05.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi032"}]: dispatch 2024-05-03T08:51:05.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: mon.smithi032 calling monitor election 2024-05-03T08:51:05.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-05-03T08:51:05.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: from='mgr.? 172.21.15.123:0/122480389' entity='mgr.smithi123.xrxinh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi123.xrxinh/crt"}]: dispatch 2024-05-03T08:51:05.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-05-03T08:51:05.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:05.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-05-03T08:51:05.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: mon.smithi123 calling monitor election 2024-05-03T08:51:05.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-05-03T08:51:05.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:05.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-05-03T08:51:05.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-05-03T08:51:05.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: mon.smithi032 is new leader, mons smithi032,smithi123 in quorum (ranks 0,1) 2024-05-03T08:51:05.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: monmap e2: 2 mons at {smithi032=[v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0],smithi123=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0]} 2024-05-03T08:51:05.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: fsmap 2024-05-03T08:51:05.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: osdmap e5: 0 total, 0 up, 0 in 2024-05-03T08:51:05.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: mgrmap e17: smithi032.umllay(active, since 24s) 2024-05-03T08:51:05.390 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: Standby manager daemon smithi123.xrxinh started 2024-05-03T08:51:05.390 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: from='mgr.? 172.21.15.123:0/122480389' entity='mgr.smithi123.xrxinh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-03T08:51:05.390 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: overall HEALTH_OK 2024-05-03T08:51:05.390 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: from='mgr.? 172.21.15.123:0/122480389' entity='mgr.smithi123.xrxinh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi123.xrxinh/key"}]: dispatch 2024-05-03T08:51:05.390 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:05.390 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:04 smithi032 ceph-mon[22403]: from='mgr.? 172.21.15.123:0/122480389' entity='mgr.smithi123.xrxinh' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-03T08:51:05.568 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-05-03T08:51:05.569 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":2,"fsid":"df63fec2-0929-11ef-bc97-c7b262605968","modified":"2024-05-03T08:50:59.845277Z","created":"2024-05-03T08:48:43.658624Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi032","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:3300","nonce":0},{"type":"v1","addr":"172.21.15.32:6789","nonce":0}]},"addr":"172.21.15.32:6789/0","public_addr":"172.21.15.32:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-05-03T08:51:05.570 INFO:teuthology.orchestra.run.smithi123.stderr:dumped monmap epoch 2 2024-05-03T08:51:06.135 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:05 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-05-03T08:51:06.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:05 smithi032 ceph-mon[22403]: Deploying daemon node-exporter.smithi123 on smithi123 2024-05-03T08:51:06.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:05 smithi032 ceph-mon[22403]: mgrmap e18: smithi032.umllay(active, since 24s), standbys: smithi123.xrxinh 2024-05-03T08:51:06.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:05 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mgr metadata", "who": "smithi123.xrxinh", "id": "smithi123.xrxinh"}]: dispatch 2024-05-03T08:51:06.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:05 smithi032 ceph-mon[22403]: from='client.? 172.21.15.123:0/2123400151' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-03T08:51:06.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:05 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-05-03T08:51:06.159 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-05-03T08:51:06.159 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph config generate-minimal-conf 2024-05-03T08:51:07.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:06 smithi032 ceph-mon[22403]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:07.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:06 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:07.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:06 smithi123 ceph-mon[24496]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:07.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:06 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:08.060 INFO:teuthology.orchestra.run.smithi032.stdout:# minimal ceph.conf for df63fec2-0929-11ef-bc97-c7b262605968 2024-05-03T08:51:08.060 INFO:teuthology.orchestra.run.smithi032.stdout:[global] 2024-05-03T08:51:08.060 INFO:teuthology.orchestra.run.smithi032.stdout: fsid = df63fec2-0929-11ef-bc97-c7b262605968 2024-05-03T08:51:08.060 INFO:teuthology.orchestra.run.smithi032.stdout: mon_host = [v2:172.21.15.32:3300/0,v1:172.21.15.32:6789/0] [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] 2024-05-03T08:51:08.703 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-05-03T08:51:08.703 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-05-03T08:51:08.703 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/etc/ceph/ceph.conf 2024-05-03T08:51:08.729 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-05-03T08:51:08.729 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-03T08:51:08.794 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-05-03T08:51:08.794 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/etc/ceph/ceph.conf 2024-05-03T08:51:08.823 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-05-03T08:51:08.823 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-03T08:51:08.887 INFO:tasks.cephadm:Deploying OSDs... 2024-05-03T08:51:08.887 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-05-03T08:51:08.887 DEBUG:teuthology.orchestra.run.smithi032:> dd if=/scratch_devs of=/dev/stdout 2024-05-03T08:51:08.902 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-05-03T08:51:08.903 DEBUG:teuthology.orchestra.run.smithi032:> stat /dev/vg_nvme/lv_1 2024-05-03T08:51:08.959 INFO:teuthology.orchestra.run.smithi032.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-05-03T08:51:08.959 INFO:teuthology.orchestra.run.smithi032.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-03T08:51:08.960 INFO:teuthology.orchestra.run.smithi032.stdout:Device: 5h/5d Inode: 664 Links: 1 2024-05-03T08:51:08.960 INFO:teuthology.orchestra.run.smithi032.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-03T08:51:08.960 INFO:teuthology.orchestra.run.smithi032.stdout:Context: system_u:object_r:device_t:s0 2024-05-03T08:51:08.960 INFO:teuthology.orchestra.run.smithi032.stdout:Access: 2024-05-03 08:50:31.083538945 +0000 2024-05-03T08:51:08.960 INFO:teuthology.orchestra.run.smithi032.stdout:Modify: 2024-05-03 08:47:53.068493057 +0000 2024-05-03T08:51:08.960 INFO:teuthology.orchestra.run.smithi032.stdout:Change: 2024-05-03 08:47:53.068493057 +0000 2024-05-03T08:51:08.960 INFO:teuthology.orchestra.run.smithi032.stdout: Birth: 2024-05-03 08:47:53.068493057 +0000 2024-05-03T08:51:08.960 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-05-03T08:51:09.021 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:08 smithi032 ceph-mon[22403]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:09.021 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:08 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/805762985' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:09.021 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:08 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:09.023 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records in 2024-05-03T08:51:09.023 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records out 2024-05-03T08:51:09.024 INFO:teuthology.orchestra.run.smithi032.stderr:512 bytes copied, 0.000123116 s, 4.2 MB/s 2024-05-03T08:51:09.025 DEBUG:teuthology.orchestra.run.smithi032:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-05-03T08:51:09.081 DEBUG:teuthology.orchestra.run.smithi032:> stat /dev/vg_nvme/lv_2 2024-05-03T08:51:09.137 INFO:teuthology.orchestra.run.smithi032.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-05-03T08:51:09.137 INFO:teuthology.orchestra.run.smithi032.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-03T08:51:09.137 INFO:teuthology.orchestra.run.smithi032.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-05-03T08:51:09.137 INFO:teuthology.orchestra.run.smithi032.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-03T08:51:09.137 INFO:teuthology.orchestra.run.smithi032.stdout:Context: system_u:object_r:device_t:s0 2024-05-03T08:51:09.137 INFO:teuthology.orchestra.run.smithi032.stdout:Access: 2024-05-03 08:50:31.083538945 +0000 2024-05-03T08:51:09.137 INFO:teuthology.orchestra.run.smithi032.stdout:Modify: 2024-05-03 08:47:53.417485767 +0000 2024-05-03T08:51:09.137 INFO:teuthology.orchestra.run.smithi032.stdout:Change: 2024-05-03 08:47:53.417485767 +0000 2024-05-03T08:51:09.137 INFO:teuthology.orchestra.run.smithi032.stdout: Birth: 2024-05-03 08:47:53.417485767 +0000 2024-05-03T08:51:09.137 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-05-03T08:51:09.202 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records in 2024-05-03T08:51:09.202 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records out 2024-05-03T08:51:09.203 INFO:teuthology.orchestra.run.smithi032.stderr:512 bytes copied, 0.000110388 s, 4.6 MB/s 2024-05-03T08:51:09.204 DEBUG:teuthology.orchestra.run.smithi032:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-05-03T08:51:09.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:08 smithi123 ceph-mon[24496]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:09.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:08 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/805762985' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:09.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:08 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:09.262 DEBUG:teuthology.orchestra.run.smithi032:> stat /dev/vg_nvme/lv_3 2024-05-03T08:51:09.317 INFO:teuthology.orchestra.run.smithi032.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-05-03T08:51:09.317 INFO:teuthology.orchestra.run.smithi032.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-03T08:51:09.318 INFO:teuthology.orchestra.run.smithi032.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-05-03T08:51:09.318 INFO:teuthology.orchestra.run.smithi032.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-03T08:51:09.318 INFO:teuthology.orchestra.run.smithi032.stdout:Context: system_u:object_r:device_t:s0 2024-05-03T08:51:09.318 INFO:teuthology.orchestra.run.smithi032.stdout:Access: 2024-05-03 08:50:31.083538945 +0000 2024-05-03T08:51:09.318 INFO:teuthology.orchestra.run.smithi032.stdout:Modify: 2024-05-03 08:47:53.757478663 +0000 2024-05-03T08:51:09.318 INFO:teuthology.orchestra.run.smithi032.stdout:Change: 2024-05-03 08:47:53.757478663 +0000 2024-05-03T08:51:09.318 INFO:teuthology.orchestra.run.smithi032.stdout: Birth: 2024-05-03 08:47:53.757478663 +0000 2024-05-03T08:51:09.318 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-05-03T08:51:09.381 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records in 2024-05-03T08:51:09.381 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records out 2024-05-03T08:51:09.382 INFO:teuthology.orchestra.run.smithi032.stderr:512 bytes copied, 0.000106564 s, 4.8 MB/s 2024-05-03T08:51:09.383 DEBUG:teuthology.orchestra.run.smithi032:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-05-03T08:51:09.439 DEBUG:teuthology.orchestra.run.smithi032:> stat /dev/vg_nvme/lv_4 2024-05-03T08:51:09.494 INFO:teuthology.orchestra.run.smithi032.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-05-03T08:51:09.494 INFO:teuthology.orchestra.run.smithi032.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-03T08:51:09.495 INFO:teuthology.orchestra.run.smithi032.stdout:Device: 5h/5d Inode: 721 Links: 1 2024-05-03T08:51:09.495 INFO:teuthology.orchestra.run.smithi032.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-03T08:51:09.495 INFO:teuthology.orchestra.run.smithi032.stdout:Context: system_u:object_r:device_t:s0 2024-05-03T08:51:09.495 INFO:teuthology.orchestra.run.smithi032.stdout:Access: 2024-05-03 08:50:31.084538925 +0000 2024-05-03T08:51:09.495 INFO:teuthology.orchestra.run.smithi032.stdout:Modify: 2024-05-03 08:47:54.102471453 +0000 2024-05-03T08:51:09.495 INFO:teuthology.orchestra.run.smithi032.stdout:Change: 2024-05-03 08:47:54.102471453 +0000 2024-05-03T08:51:09.495 INFO:teuthology.orchestra.run.smithi032.stdout: Birth: 2024-05-03 08:47:54.102471453 +0000 2024-05-03T08:51:09.495 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-05-03T08:51:09.556 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records in 2024-05-03T08:51:09.556 INFO:teuthology.orchestra.run.smithi032.stderr:1+0 records out 2024-05-03T08:51:09.557 INFO:teuthology.orchestra.run.smithi032.stderr:512 bytes copied, 0.000113952 s, 4.5 MB/s 2024-05-03T08:51:09.558 DEBUG:teuthology.orchestra.run.smithi032:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-05-03T08:51:09.614 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-05-03T08:51:09.614 DEBUG:teuthology.orchestra.run.smithi123:> dd if=/scratch_devs of=/dev/stdout 2024-05-03T08:51:09.633 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-05-03T08:51:09.633 DEBUG:teuthology.orchestra.run.smithi123:> stat /dev/vg_nvme/lv_1 2024-05-03T08:51:09.691 INFO:teuthology.orchestra.run.smithi123.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-05-03T08:51:09.691 INFO:teuthology.orchestra.run.smithi123.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-03T08:51:09.691 INFO:teuthology.orchestra.run.smithi123.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-05-03T08:51:09.691 INFO:teuthology.orchestra.run.smithi123.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-03T08:51:09.691 INFO:teuthology.orchestra.run.smithi123.stdout:Context: system_u:object_r:device_t:s0 2024-05-03T08:51:09.691 INFO:teuthology.orchestra.run.smithi123.stdout:Access: 2024-05-03 08:51:05.455731437 +0000 2024-05-03T08:51:09.691 INFO:teuthology.orchestra.run.smithi123.stdout:Modify: 2024-05-03 08:47:41.462309708 +0000 2024-05-03T08:51:09.692 INFO:teuthology.orchestra.run.smithi123.stdout:Change: 2024-05-03 08:47:41.462309708 +0000 2024-05-03T08:51:09.692 INFO:teuthology.orchestra.run.smithi123.stdout: Birth: 2024-05-03 08:47:41.462309708 +0000 2024-05-03T08:51:09.692 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-05-03T08:51:09.758 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records in 2024-05-03T08:51:09.758 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records out 2024-05-03T08:51:09.758 INFO:teuthology.orchestra.run.smithi123.stderr:512 bytes copied, 0.000240985 s, 2.1 MB/s 2024-05-03T08:51:09.759 DEBUG:teuthology.orchestra.run.smithi123:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-05-03T08:51:09.818 DEBUG:teuthology.orchestra.run.smithi123:> stat /dev/vg_nvme/lv_2 2024-05-03T08:51:09.876 INFO:teuthology.orchestra.run.smithi123.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-05-03T08:51:09.876 INFO:teuthology.orchestra.run.smithi123.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-03T08:51:09.876 INFO:teuthology.orchestra.run.smithi123.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-05-03T08:51:09.876 INFO:teuthology.orchestra.run.smithi123.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-03T08:51:09.877 INFO:teuthology.orchestra.run.smithi123.stdout:Context: system_u:object_r:device_t:s0 2024-05-03T08:51:09.877 INFO:teuthology.orchestra.run.smithi123.stdout:Access: 2024-05-03 08:51:05.455731437 +0000 2024-05-03T08:51:09.877 INFO:teuthology.orchestra.run.smithi123.stdout:Modify: 2024-05-03 08:47:41.804301516 +0000 2024-05-03T08:51:09.877 INFO:teuthology.orchestra.run.smithi123.stdout:Change: 2024-05-03 08:47:41.804301516 +0000 2024-05-03T08:51:09.877 INFO:teuthology.orchestra.run.smithi123.stdout: Birth: 2024-05-03 08:47:41.804301516 +0000 2024-05-03T08:51:09.877 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-05-03T08:51:09.943 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records in 2024-05-03T08:51:09.943 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records out 2024-05-03T08:51:09.943 INFO:teuthology.orchestra.run.smithi123.stderr:512 bytes copied, 0.000236747 s, 2.2 MB/s 2024-05-03T08:51:09.944 DEBUG:teuthology.orchestra.run.smithi123:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-05-03T08:51:10.003 DEBUG:teuthology.orchestra.run.smithi123:> stat /dev/vg_nvme/lv_3 2024-05-03T08:51:10.061 INFO:teuthology.orchestra.run.smithi123.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-05-03T08:51:10.061 INFO:teuthology.orchestra.run.smithi123.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-03T08:51:10.061 INFO:teuthology.orchestra.run.smithi123.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-05-03T08:51:10.061 INFO:teuthology.orchestra.run.smithi123.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-03T08:51:10.061 INFO:teuthology.orchestra.run.smithi123.stdout:Context: system_u:object_r:device_t:s0 2024-05-03T08:51:10.061 INFO:teuthology.orchestra.run.smithi123.stdout:Access: 2024-05-03 08:51:05.455731437 +0000 2024-05-03T08:51:10.062 INFO:teuthology.orchestra.run.smithi123.stdout:Modify: 2024-05-03 08:47:42.163292916 +0000 2024-05-03T08:51:10.062 INFO:teuthology.orchestra.run.smithi123.stdout:Change: 2024-05-03 08:47:42.163292916 +0000 2024-05-03T08:51:10.062 INFO:teuthology.orchestra.run.smithi123.stdout: Birth: 2024-05-03 08:47:42.163292916 +0000 2024-05-03T08:51:10.062 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-05-03T08:51:10.125 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:09 smithi123 ceph-mon[24496]: Deploying daemon prometheus.smithi032 on smithi032 2024-05-03T08:51:10.127 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records in 2024-05-03T08:51:10.127 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records out 2024-05-03T08:51:10.128 INFO:teuthology.orchestra.run.smithi123.stderr:512 bytes copied, 0.000236119 s, 2.2 MB/s 2024-05-03T08:51:10.129 DEBUG:teuthology.orchestra.run.smithi123:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-05-03T08:51:10.188 DEBUG:teuthology.orchestra.run.smithi123:> stat /dev/vg_nvme/lv_4 2024-05-03T08:51:10.245 INFO:teuthology.orchestra.run.smithi123.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-05-03T08:51:10.246 INFO:teuthology.orchestra.run.smithi123.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-03T08:51:10.246 INFO:teuthology.orchestra.run.smithi123.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-05-03T08:51:10.246 INFO:teuthology.orchestra.run.smithi123.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-03T08:51:10.246 INFO:teuthology.orchestra.run.smithi123.stdout:Context: system_u:object_r:device_t:s0 2024-05-03T08:51:10.246 INFO:teuthology.orchestra.run.smithi123.stdout:Access: 2024-05-03 08:51:05.456731413 +0000 2024-05-03T08:51:10.246 INFO:teuthology.orchestra.run.smithi123.stdout:Modify: 2024-05-03 08:47:42.513284532 +0000 2024-05-03T08:51:10.246 INFO:teuthology.orchestra.run.smithi123.stdout:Change: 2024-05-03 08:47:42.513284532 +0000 2024-05-03T08:51:10.246 INFO:teuthology.orchestra.run.smithi123.stdout: Birth: 2024-05-03 08:47:42.513284532 +0000 2024-05-03T08:51:10.246 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-05-03T08:51:10.312 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records in 2024-05-03T08:51:10.312 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records out 2024-05-03T08:51:10.312 INFO:teuthology.orchestra.run.smithi123.stderr:512 bytes copied, 0.000235115 s, 2.2 MB/s 2024-05-03T08:51:10.314 DEBUG:teuthology.orchestra.run.smithi123:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-05-03T08:51:10.372 INFO:tasks.cephadm:Deploying osd.0 on smithi032 with /dev/vg_nvme/lv_4... 2024-05-03T08:51:10.373 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-05-03T08:51:10.387 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:09 smithi032 ceph-mon[22403]: Deploying daemon prometheus.smithi032 on smithi032 2024-05-03T08:51:11.121 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:10 smithi032 ceph-mon[22403]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:11.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:10 smithi123 ceph-mon[24496]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:12.830 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:12 smithi032 ceph-mon[22403]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:12.831 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:12 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:13.094 INFO:teuthology.orchestra.run.smithi032.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-05-03T08:51:13.095 INFO:teuthology.orchestra.run.smithi032.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-05-03T08:51:13.095 INFO:teuthology.orchestra.run.smithi032.stderr: stderr: 10+0 records in 2024-05-03T08:51:13.095 INFO:teuthology.orchestra.run.smithi032.stderr:10+0 records out 2024-05-03T08:51:13.095 INFO:teuthology.orchestra.run.smithi032.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00953252 s, 1.1 GB/s 2024-05-03T08:51:13.095 INFO:teuthology.orchestra.run.smithi032.stderr:--> Zapping successful for: 2024-05-03T08:51:13.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:12 smithi123 ceph-mon[24496]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:13.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:12 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:14.315 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi032:vg_nvme/lv_4 2024-05-03T08:51:15.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:14 smithi032 ceph-mon[22403]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:15.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:14 smithi123 ceph-mon[24496]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:16.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:16 smithi123 ceph-mon[24496]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:17.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:16 smithi032 ceph-mon[22403]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:18.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:18 smithi123 ceph-mon[24496]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:18.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:18 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-03T08:51:18.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:18 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-03T08:51:18.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:18 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:19.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:18 smithi032 ceph-mon[22403]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:19.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:18 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-03T08:51:19.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:18 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-03T08:51:19.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:18 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:19.990 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:19 smithi032 ceph-mon[22403]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:51:20.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:19 smithi123 ceph-mon[24496]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:51:20.845 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:20 smithi032 ceph-mon[22403]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:21.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:20 smithi123 ceph-mon[24496]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:23.006 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:22 smithi032 ceph-mon[22403]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:23.166 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:22 smithi123 ceph-mon[24496]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:24.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:24 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:24.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:24 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:51:24.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:24 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:24.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:24 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:51:24.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:24 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:24.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:24 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:51:24.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:24 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:24.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:24 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:51:25.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:25 smithi032 ceph-mon[22403]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:25.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:25 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/569806838' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "39f89f27-f5a3-4b54-8b6e-cb5e8ef45738"}]: dispatch 2024-05-03T08:51:25.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:25 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/569806838' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "39f89f27-f5a3-4b54-8b6e-cb5e8ef45738"}]': finished 2024-05-03T08:51:25.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:25 smithi032 ceph-mon[22403]: osdmap e6: 1 total, 0 up, 1 in 2024-05-03T08:51:25.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:25 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-03T08:51:25.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:25 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:25.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:25 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/2545547909' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-03T08:51:25.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:25 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:25.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:25 smithi123 ceph-mon[24496]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:25.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:25 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/569806838' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "39f89f27-f5a3-4b54-8b6e-cb5e8ef45738"}]: dispatch 2024-05-03T08:51:25.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:25 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/569806838' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "39f89f27-f5a3-4b54-8b6e-cb5e8ef45738"}]': finished 2024-05-03T08:51:25.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:25 smithi123 ceph-mon[24496]: osdmap e6: 1 total, 0 up, 1 in 2024-05-03T08:51:25.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:25 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-03T08:51:25.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:25 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:25.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:25 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/2545547909' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-03T08:51:25.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:25 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:26.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:26 smithi123 ceph-mon[24496]: Updating smithi123:/etc/ceph/ceph.conf 2024-05-03T08:51:26.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:26 smithi123 ceph-mon[24496]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2024-05-03T08:51:26.523 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:26 smithi032 ceph-mon[22403]: Updating smithi123:/etc/ceph/ceph.conf 2024-05-03T08:51:26.523 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:26 smithi032 ceph-mon[22403]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2024-05-03T08:51:27.410 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:27 smithi032 ceph-mon[22403]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:27.410 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:27 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:27.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:27 smithi123 ceph-mon[24496]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:27.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:27 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:28.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:28 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:28.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:28 smithi032 ceph-mon[22403]: Updating smithi032:/etc/ceph/ceph.conf 2024-05-03T08:51:28.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:28 smithi032 ceph-mon[22403]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:28.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:28 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:28.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:28 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:28.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:28 smithi032 ceph-mon[22403]: Reconfiguring mon.smithi032 (unknown last config time)... 2024-05-03T08:51:28.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:28 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-03T08:51:28.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:28 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-03T08:51:28.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:28 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:28.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:28 smithi032 ceph-mon[22403]: Reconfiguring daemon mon.smithi032 on smithi032 2024-05-03T08:51:28.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:28 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:28.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:28 smithi123 ceph-mon[24496]: Updating smithi032:/etc/ceph/ceph.conf 2024-05-03T08:51:28.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:28 smithi123 ceph-mon[24496]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:28.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:28 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:28.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:28 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:28.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:28 smithi123 ceph-mon[24496]: Reconfiguring mon.smithi032 (unknown last config time)... 2024-05-03T08:51:28.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:28 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-03T08:51:28.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:28 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-03T08:51:28.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:28 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:28.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:28 smithi123 ceph-mon[24496]: Reconfiguring daemon mon.smithi032 on smithi032 2024-05-03T08:51:30.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:30 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:30.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:30 smithi032 ceph-mon[22403]: Reconfiguring mgr.smithi032.umllay (unknown last config time)... 2024-05-03T08:51:30.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:30 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi032.umllay", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-03T08:51:30.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:30 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-03T08:51:30.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:30 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:30.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:30 smithi032 ceph-mon[22403]: Reconfiguring daemon mgr.smithi032.umllay on smithi032 2024-05-03T08:51:30.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:30 smithi032 ceph-mon[22403]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:30.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:30 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:30.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:30 smithi123 ceph-mon[24496]: Reconfiguring mgr.smithi032.umllay (unknown last config time)... 2024-05-03T08:51:30.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:30 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi032.umllay", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-03T08:51:30.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:30 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-03T08:51:30.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:30 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:30.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:30 smithi123 ceph-mon[24496]: Reconfiguring daemon mgr.smithi032.umllay on smithi032 2024-05-03T08:51:30.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:30 smithi123 ceph-mon[24496]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:33.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:32 smithi032 ceph-mon[22403]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:33.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:33.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:32 smithi032 ceph-mon[22403]: Reconfiguring alertmanager.smithi032 (dependencies changed)... 2024-05-03T08:51:33.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:32 smithi032 ceph-mon[22403]: Reconfiguring daemon alertmanager.smithi032 on smithi032 2024-05-03T08:51:33.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:32 smithi123 ceph-mon[24496]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:33.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:33.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:32 smithi123 ceph-mon[24496]: Reconfiguring alertmanager.smithi032 (dependencies changed)... 2024-05-03T08:51:33.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:32 smithi123 ceph-mon[24496]: Reconfiguring daemon alertmanager.smithi032 on smithi032 2024-05-03T08:51:34.873 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:34 smithi032 ceph-mon[22403]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:34.873 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:34 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-03T08:51:34.873 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:34 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:34.873 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:34 smithi032 ceph-mon[22403]: Deploying daemon osd.0 on smithi032 2024-05-03T08:51:35.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:34 smithi123 ceph-mon[24496]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:35.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:34 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-03T08:51:35.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:34 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:35.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:34 smithi123 ceph-mon[24496]: Deploying daemon osd.0 on smithi032 2024-05-03T08:51:36.964 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:36 smithi032 ceph-mon[22403]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:37.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:36 smithi123 ceph-mon[24496]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:38.804 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:38 smithi032 ceph-mon[22403]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:38.805 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:38 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:38.805 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:38 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi032", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-03T08:51:38.805 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:38 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:39.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:38 smithi123 ceph-mon[24496]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:39.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:38 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:39.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:38 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi032", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-03T08:51:39.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:38 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:40.027 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:39 smithi032 ceph-mon[22403]: Reconfiguring crash.smithi032 (monmap changed)... 2024-05-03T08:51:40.027 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:39 smithi032 ceph-mon[22403]: Reconfiguring daemon crash.smithi032 on smithi032 2024-05-03T08:51:40.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:39 smithi123 ceph-mon[24496]: Reconfiguring crash.smithi032 (monmap changed)... 2024-05-03T08:51:40.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:39 smithi123 ceph-mon[24496]: Reconfiguring daemon crash.smithi032 on smithi032 2024-05-03T08:51:41.097 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:40 smithi032 ceph-mon[22403]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:41.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:40 smithi123 ceph-mon[24496]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:43.081 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:42 smithi032 ceph-mon[22403]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:43.081 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:51:43.081 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:51:43.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:42 smithi123 ceph-mon[24496]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:43.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:51:43.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:51:44.862 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:44 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:44.862 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:44 smithi032 ceph-mon[22403]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:44.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:44 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:44.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:44 smithi123 ceph-mon[24496]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:46.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:45 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:46.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:45 smithi032 ceph-mon[22403]: Reconfiguring grafana.smithi032 (dependencies changed)... 2024-05-03T08:51:46.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:45 smithi032 ceph-mon[22403]: Reconfiguring daemon grafana.smithi032 on smithi032 2024-05-03T08:51:46.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:45 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:46.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:45 smithi123 ceph-mon[24496]: Reconfiguring grafana.smithi032 (dependencies changed)... 2024-05-03T08:51:46.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:45 smithi123 ceph-mon[24496]: Reconfiguring daemon grafana.smithi032 on smithi032 2024-05-03T08:51:47.098 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:46 smithi032 ceph-mon[22403]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:47.099 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:46 smithi032 ceph-mon[22403]: from='osd.0 [v2:172.21.15.32:6802/2936167065,v1:172.21.15.32:6803/2936167065]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-03T08:51:47.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:46 smithi123 ceph-mon[24496]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:47.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:46 smithi123 ceph-mon[24496]: from='osd.0 [v2:172.21.15.32:6802/2936167065,v1:172.21.15.32:6803/2936167065]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-03T08:51:49.026 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:48 smithi032 ceph-mon[22403]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:49.026 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:48 smithi032 ceph-mon[22403]: from='osd.0 [v2:172.21.15.32:6802/2936167065,v1:172.21.15.32:6803/2936167065]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-03T08:51:49.026 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:48 smithi032 ceph-mon[22403]: osdmap e7: 1 total, 0 up, 1 in 2024-05-03T08:51:49.026 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:48 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-03T08:51:49.026 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:48 smithi032 ceph-mon[22403]: from='osd.0 [v2:172.21.15.32:6802/2936167065,v1:172.21.15.32:6803/2936167065]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-05-03T08:51:49.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:48 smithi123 ceph-mon[24496]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:49.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:48 smithi123 ceph-mon[24496]: from='osd.0 [v2:172.21.15.32:6802/2936167065,v1:172.21.15.32:6803/2936167065]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-03T08:51:49.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:48 smithi123 ceph-mon[24496]: osdmap e7: 1 total, 0 up, 1 in 2024-05-03T08:51:49.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:48 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-03T08:51:49.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:48 smithi123 ceph-mon[24496]: from='osd.0 [v2:172.21.15.32:6802/2936167065,v1:172.21.15.32:6803/2936167065]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-05-03T08:51:49.874 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:49 smithi032 ceph-mon[22403]: from='osd.0 [v2:172.21.15.32:6802/2936167065,v1:172.21.15.32:6803/2936167065]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-05-03T08:51:49.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:49 smithi032 ceph-mon[22403]: osdmap e8: 1 total, 0 up, 1 in 2024-05-03T08:51:49.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-03T08:51:49.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-03T08:51:49.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:49.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:49 smithi032 ceph-mon[22403]: Reconfiguring crash.smithi123 (monmap changed)... 2024-05-03T08:51:49.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-03T08:51:49.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:49.875 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:49 smithi032 ceph-mon[22403]: Reconfiguring daemon crash.smithi123 on smithi123 2024-05-03T08:51:50.060 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:49 smithi123 ceph-mon[24496]: from='osd.0 [v2:172.21.15.32:6802/2936167065,v1:172.21.15.32:6803/2936167065]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-05-03T08:51:50.060 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:49 smithi123 ceph-mon[24496]: osdmap e8: 1 total, 0 up, 1 in 2024-05-03T08:51:50.060 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-03T08:51:50.060 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-03T08:51:50.060 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:50.060 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:49 smithi123 ceph-mon[24496]: Reconfiguring crash.smithi123 (monmap changed)... 2024-05-03T08:51:50.061 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-03T08:51:50.061 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:50.061 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:49 smithi123 ceph-mon[24496]: Reconfiguring daemon crash.smithi123 on smithi123 2024-05-03T08:51:50.845 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:50 smithi032 ceph-mon[22403]: purged_snaps scrub starts 2024-05-03T08:51:50.845 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:50 smithi032 ceph-mon[22403]: purged_snaps scrub ok 2024-05-03T08:51:50.846 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:50 smithi032 ceph-mon[22403]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:50.846 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:50 smithi032 ceph-mon[22403]: osd.0 [v2:172.21.15.32:6802/2936167065,v1:172.21.15.32:6803/2936167065] boot 2024-05-03T08:51:50.846 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:50 smithi032 ceph-mon[22403]: osdmap e9: 1 total, 1 up, 1 in 2024-05-03T08:51:50.846 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:50 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-03T08:51:50.846 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:50 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:50.846 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:50 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi123.xrxinh", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-03T08:51:50.846 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:50 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-03T08:51:50.846 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:50 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:50.876 INFO:teuthology.orchestra.run.smithi032.stdout:Created osd(s) 0 on host 'smithi032' 2024-05-03T08:51:50.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:50 smithi123 ceph-mon[24496]: purged_snaps scrub starts 2024-05-03T08:51:50.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:50 smithi123 ceph-mon[24496]: purged_snaps scrub ok 2024-05-03T08:51:50.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:50 smithi123 ceph-mon[24496]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-03T08:51:50.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:50 smithi123 ceph-mon[24496]: osd.0 [v2:172.21.15.32:6802/2936167065,v1:172.21.15.32:6803/2936167065] boot 2024-05-03T08:51:50.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:50 smithi123 ceph-mon[24496]: osdmap e9: 1 total, 1 up, 1 in 2024-05-03T08:51:50.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:50 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-03T08:51:50.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:50 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:50.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:50 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi123.xrxinh", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-03T08:51:50.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:50 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-03T08:51:50.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:50 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:51.507 DEBUG:teuthology.orchestra.run.smithi032:osd.0> sudo journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.0.service 2024-05-03T08:51:51.510 INFO:tasks.cephadm:Deploying osd.1 on smithi032 with /dev/vg_nvme/lv_3... 2024-05-03T08:51:51.510 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-05-03T08:51:52.006 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:51 smithi123 ceph-mon[24496]: Reconfiguring mgr.smithi123.xrxinh (monmap changed)... 2024-05-03T08:51:52.006 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:51 smithi123 ceph-mon[24496]: Reconfiguring daemon mgr.smithi123.xrxinh on smithi123 2024-05-03T08:51:52.006 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:51 smithi123 ceph-mon[24496]: osdmap e10: 1 total, 1 up, 1 in 2024-05-03T08:51:52.007 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:51 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:52.015 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:51 smithi032 ceph-mon[22403]: Reconfiguring mgr.smithi123.xrxinh (monmap changed)... 2024-05-03T08:51:52.015 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:51 smithi032 ceph-mon[22403]: Reconfiguring daemon mgr.smithi123.xrxinh on smithi123 2024-05-03T08:51:52.015 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:51 smithi032 ceph-mon[22403]: osdmap e10: 1 total, 1 up, 1 in 2024-05-03T08:51:52.016 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:51 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:53.268 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:53 smithi032 ceph-mon[22403]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:51:53.268 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:53 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:53.268 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:53 smithi032 ceph-mon[22403]: Reconfiguring mon.smithi123 (monmap changed)... 2024-05-03T08:51:53.268 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:53 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-03T08:51:53.269 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:53 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-03T08:51:53.269 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:53 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:53.269 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:53 smithi032 ceph-mon[22403]: Reconfiguring daemon mon.smithi123 on smithi123 2024-05-03T08:51:53.269 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:53 smithi123 ceph-mon[24496]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:51:53.269 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:53 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:53.269 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:53 smithi123 ceph-mon[24496]: Reconfiguring mon.smithi123 (monmap changed)... 2024-05-03T08:51:53.269 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:53 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-03T08:51:53.269 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:53 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-03T08:51:53.270 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:53 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:53.270 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:53 smithi123 ceph-mon[24496]: Reconfiguring daemon mon.smithi123 on smithi123 2024-05-03T08:51:53.569 INFO:teuthology.orchestra.run.smithi032.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-05-03T08:51:53.569 INFO:teuthology.orchestra.run.smithi032.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-05-03T08:51:53.569 INFO:teuthology.orchestra.run.smithi032.stderr: stderr: 10+0 records in 2024-05-03T08:51:53.569 INFO:teuthology.orchestra.run.smithi032.stderr:10+0 records out 2024-05-03T08:51:53.569 INFO:teuthology.orchestra.run.smithi032.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0102278 s, 1.0 GB/s 2024-05-03T08:51:53.569 INFO:teuthology.orchestra.run.smithi032.stderr:--> Zapping successful for: 2024-05-03T08:51:54.209 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi032:vg_nvme/lv_3 2024-05-03T08:51:54.468 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:54 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:54.468 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:54 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-03T08:51:54.468 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:54 smithi032 ceph-mon[22403]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-03T08:51:54.468 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:54 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.32:9093"}]: dispatch 2024-05-03T08:51:54.468 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:54 smithi032 ceph-mon[22403]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.32:9093"}]: dispatch 2024-05-03T08:51:54.469 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:54 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:54.469 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:54 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-03T08:51:54.469 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:54 smithi032 ceph-mon[22403]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-03T08:51:54.469 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:54 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.32:3000"}]: dispatch 2024-05-03T08:51:54.469 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:54 smithi032 ceph-mon[22403]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.32:3000"}]: dispatch 2024-05-03T08:51:54.469 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:54 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:54.469 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:54 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-03T08:51:54.469 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:54 smithi032 ceph-mon[22403]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-03T08:51:54.469 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:54 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.32:9095"}]: dispatch 2024-05-03T08:51:54.469 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:54 smithi032 ceph-mon[22403]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.32:9095"}]: dispatch 2024-05-03T08:51:54.469 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:54 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:54.469 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:54 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:51:54.469 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:54 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:54.469 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:54 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:51:54.619 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:54 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:54.619 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:54 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-03T08:51:54.619 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:54 smithi123 ceph-mon[24496]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-03T08:51:54.619 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:54 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.32:9093"}]: dispatch 2024-05-03T08:51:54.619 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:54 smithi123 ceph-mon[24496]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.32:9093"}]: dispatch 2024-05-03T08:51:54.619 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:54 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:54.619 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:54 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-03T08:51:54.619 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:54 smithi123 ceph-mon[24496]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-03T08:51:54.619 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:54 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.32:3000"}]: dispatch 2024-05-03T08:51:54.619 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:54 smithi123 ceph-mon[24496]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.32:3000"}]: dispatch 2024-05-03T08:51:54.619 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:54 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:54.619 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:54 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-03T08:51:54.619 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:54 smithi123 ceph-mon[24496]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-03T08:51:54.619 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:54 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.32:9095"}]: dispatch 2024-05-03T08:51:54.620 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:54 smithi123 ceph-mon[24496]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.32:9095"}]: dispatch 2024-05-03T08:51:54.620 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:54 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:54.620 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:54 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:51:54.620 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:54 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:54.620 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:54 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:51:55.504 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:55 smithi032 ceph-mon[22403]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:51:55.504 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:55 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:55.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:55 smithi123 ceph-mon[24496]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:51:55.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:55 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:51:57.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:56 smithi032 ceph-mon[22403]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:51:57.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:56 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-03T08:51:57.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:56 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-03T08:51:57.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:56 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:57.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:56 smithi123 ceph-mon[24496]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:51:57.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:56 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-03T08:51:57.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:56 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-03T08:51:57.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:56 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:51:58.028 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:57 smithi032 ceph-mon[22403]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:51:58.035 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:57 smithi123 ceph-mon[24496]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:51:58.869 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:51:58 smithi032 ceph-mon[22403]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:51:59.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:51:58 smithi123 ceph-mon[24496]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:01.029 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:00 smithi123 ceph-mon[24496]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:01.029 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:00 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/1516050890' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2bc68418-2844-49df-8c1f-016dbbe3d34d"}]: dispatch 2024-05-03T08:52:01.029 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:00 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/1516050890' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2bc68418-2844-49df-8c1f-016dbbe3d34d"}]': finished 2024-05-03T08:52:01.029 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:00 smithi123 ceph-mon[24496]: osdmap e11: 2 total, 1 up, 2 in 2024-05-03T08:52:01.029 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:00 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-03T08:52:01.029 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:00 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:01.029 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:00 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/449438860' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-03T08:52:01.092 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:00 smithi032 ceph-mon[22403]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:01.092 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:00 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/1516050890' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2bc68418-2844-49df-8c1f-016dbbe3d34d"}]: dispatch 2024-05-03T08:52:01.092 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:00 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/1516050890' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2bc68418-2844-49df-8c1f-016dbbe3d34d"}]': finished 2024-05-03T08:52:01.092 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:00 smithi032 ceph-mon[22403]: osdmap e11: 2 total, 1 up, 2 in 2024-05-03T08:52:01.092 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:00 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-03T08:52:01.092 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:00 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:01.092 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:00 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/449438860' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-03T08:52:03.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:02 smithi032 ceph-mon[22403]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:03.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:02 smithi123 ceph-mon[24496]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:05.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:04 smithi032 ceph-mon[22403]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:05.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:04 smithi123 ceph-mon[24496]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:06.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:06 smithi123 ceph-mon[24496]: Detected new or changed devices on smithi032 2024-05-03T08:52:06.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:06 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:06.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:06 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:52:06.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:06 smithi123 ceph-mon[24496]: Adjusting osd_memory_target on smithi032 to 12869M 2024-05-03T08:52:06.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:06 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:06.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:06 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:06.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:06 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:52:06.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:06 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:52:06.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:06 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:52:06.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:06 smithi123 ceph-mon[24496]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:06.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:06 smithi032 ceph-mon[22403]: Detected new or changed devices on smithi032 2024-05-03T08:52:06.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:06 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:06.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:06 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:52:06.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:06 smithi032 ceph-mon[22403]: Adjusting osd_memory_target on smithi032 to 12869M 2024-05-03T08:52:06.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:06 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:06.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:06 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:06.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:06 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:52:06.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:06 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:52:06.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:06 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:52:06.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:06 smithi032 ceph-mon[22403]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:08.282 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:08 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:08.282 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:08 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-03T08:52:08.282 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:08 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:52:08.282 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:08 smithi032 ceph-mon[22403]: Deploying daemon osd.1 on smithi032 2024-05-03T08:52:08.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:08 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:08.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:08 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-03T08:52:08.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:08 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:52:08.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:08 smithi123 ceph-mon[24496]: Deploying daemon osd.1 on smithi032 2024-05-03T08:52:09.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:09 smithi032 ceph-mon[22403]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:09.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:09 smithi123 ceph-mon[24496]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:11.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:11 smithi123 ceph-mon[24496]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:11.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:11 smithi032 ceph-mon[22403]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:13.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:13 smithi123 ceph-mon[24496]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:13.478 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:13 smithi032 ceph-mon[22403]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:15.109 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:14 smithi032 ceph-mon[22403]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:15.109 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:14 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:15.109 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:14 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:15.109 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:14 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:15.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:14 smithi123 ceph-mon[24496]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:15.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:14 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:15.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:14 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:15.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:14 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:16.067 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:15 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:52:16.067 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:15 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:52:16.067 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:15 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:52:16.156 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:15 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:52:16.156 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:15 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:52:16.156 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:15 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:52:16.971 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:16 smithi032 ceph-mon[22403]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:16.972 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:16 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:16.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:16 smithi123 ceph-mon[24496]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:16.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:16 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:18.470 INFO:teuthology.orchestra.run.smithi032.stdout:Created osd(s) 1 on host 'smithi032' 2024-05-03T08:52:19.076 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:18 smithi032 ceph-mon[22403]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:19.076 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:18 smithi032 ceph-mon[22403]: from='osd.1 [v2:172.21.15.32:6810/4253870677,v1:172.21.15.32:6811/4253870677]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-03T08:52:19.076 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:18 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:19.102 DEBUG:teuthology.orchestra.run.smithi032:osd.1> sudo journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.1.service 2024-05-03T08:52:19.105 INFO:tasks.cephadm:Deploying osd.2 on smithi032 with /dev/vg_nvme/lv_2... 2024-05-03T08:52:19.105 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-05-03T08:52:19.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:18 smithi123 ceph-mon[24496]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:19.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:18 smithi123 ceph-mon[24496]: from='osd.1 [v2:172.21.15.32:6810/4253870677,v1:172.21.15.32:6811/4253870677]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-03T08:52:19.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:18 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:20.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:19 smithi032 ceph-mon[22403]: from='osd.1 [v2:172.21.15.32:6810/4253870677,v1:172.21.15.32:6811/4253870677]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-03T08:52:20.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:19 smithi032 ceph-mon[22403]: osdmap e12: 2 total, 1 up, 2 in 2024-05-03T08:52:20.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:19 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-03T08:52:20.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:19 smithi032 ceph-mon[22403]: from='osd.1 [v2:172.21.15.32:6810/4253870677,v1:172.21.15.32:6811/4253870677]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-05-03T08:52:20.138 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:52:19 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-1[36879]: 2024-05-03T08:52:19.753+0000 7f798f009700 -1 osd.1 0 waiting for initial osdmap 2024-05-03T08:52:20.139 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:52:19 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-1[36879]: 2024-05-03T08:52:19.756+0000 7f79899a0700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-03T08:52:20.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:19 smithi123 ceph-mon[24496]: from='osd.1 [v2:172.21.15.32:6810/4253870677,v1:172.21.15.32:6811/4253870677]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-03T08:52:20.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:19 smithi123 ceph-mon[24496]: osdmap e12: 2 total, 1 up, 2 in 2024-05-03T08:52:20.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:19 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-03T08:52:20.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:19 smithi123 ceph-mon[24496]: from='osd.1 [v2:172.21.15.32:6810/4253870677,v1:172.21.15.32:6811/4253870677]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-05-03T08:52:21.041 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:20 smithi032 ceph-mon[22403]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:21.041 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:20 smithi032 ceph-mon[22403]: from='osd.1 [v2:172.21.15.32:6810/4253870677,v1:172.21.15.32:6811/4253870677]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-05-03T08:52:21.041 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:20 smithi032 ceph-mon[22403]: osdmap e13: 2 total, 1 up, 2 in 2024-05-03T08:52:21.041 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:20 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-03T08:52:21.041 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:20 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-03T08:52:21.041 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:20 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-03T08:52:21.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:20 smithi123 ceph-mon[24496]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:21.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:20 smithi123 ceph-mon[24496]: from='osd.1 [v2:172.21.15.32:6810/4253870677,v1:172.21.15.32:6811/4253870677]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-05-03T08:52:21.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:20 smithi123 ceph-mon[24496]: osdmap e13: 2 total, 1 up, 2 in 2024-05-03T08:52:21.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:20 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-03T08:52:21.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:20 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-03T08:52:21.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:20 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-03T08:52:21.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:21 smithi032 ceph-mon[22403]: purged_snaps scrub starts 2024-05-03T08:52:21.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:21 smithi032 ceph-mon[22403]: purged_snaps scrub ok 2024-05-03T08:52:21.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:21 smithi032 ceph-mon[22403]: osd.1 [v2:172.21.15.32:6810/4253870677,v1:172.21.15.32:6811/4253870677] boot 2024-05-03T08:52:21.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:21 smithi032 ceph-mon[22403]: osdmap e14: 2 total, 2 up, 2 in 2024-05-03T08:52:21.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:21 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-03T08:52:21.952 INFO:teuthology.orchestra.run.smithi032.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-05-03T08:52:21.952 INFO:teuthology.orchestra.run.smithi032.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-05-03T08:52:21.953 INFO:teuthology.orchestra.run.smithi032.stderr: stderr: 10+0 records in 2024-05-03T08:52:21.953 INFO:teuthology.orchestra.run.smithi032.stderr:10+0 records out 2024-05-03T08:52:21.953 INFO:teuthology.orchestra.run.smithi032.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0114684 s, 914 MB/s 2024-05-03T08:52:21.953 INFO:teuthology.orchestra.run.smithi032.stderr: stderr: 2024-05-03T08:52:21.953 INFO:teuthology.orchestra.run.smithi032.stderr:--> Zapping successful for: 2024-05-03T08:52:22.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:21 smithi123 ceph-mon[24496]: purged_snaps scrub starts 2024-05-03T08:52:22.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:21 smithi123 ceph-mon[24496]: purged_snaps scrub ok 2024-05-03T08:52:22.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:21 smithi123 ceph-mon[24496]: osd.1 [v2:172.21.15.32:6810/4253870677,v1:172.21.15.32:6811/4253870677] boot 2024-05-03T08:52:22.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:21 smithi123 ceph-mon[24496]: osdmap e14: 2 total, 2 up, 2 in 2024-05-03T08:52:22.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:21 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-03T08:52:22.572 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi032:vg_nvme/lv_2 2024-05-03T08:52:22.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:22 smithi032 ceph-mon[22403]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:22.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:22 smithi032 ceph-mon[22403]: osdmap e15: 2 total, 2 up, 2 in 2024-05-03T08:52:23.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:22 smithi123 ceph-mon[24496]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-03T08:52:23.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:22 smithi123 ceph-mon[24496]: osdmap e15: 2 total, 2 up, 2 in 2024-05-03T08:52:25.447 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:25 smithi032 ceph-mon[22403]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:25.448 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:25 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:25.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:25 smithi123 ceph-mon[24496]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:25.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:25 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:26.428 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:26 smithi032 ceph-mon[22403]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:52:26.428 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:26 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-03T08:52:26.428 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:26 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-03T08:52:26.428 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:26 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:52:26.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:26 smithi123 ceph-mon[24496]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:52:26.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:26 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-03T08:52:26.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:26 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-03T08:52:26.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:26 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:52:27.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:27 smithi123 ceph-mon[24496]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:27.595 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:27 smithi032 ceph-mon[22403]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:29.205 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:28 smithi032 ceph-mon[22403]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:29.205 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:28 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:29.205 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:28 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:52:29.205 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:28 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:52:29.205 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:28 smithi032 ceph-mon[22403]: Adjusting osd_memory_target on smithi032 to 6434M 2024-05-03T08:52:29.205 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:28 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:29.206 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:28 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:29.206 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:28 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:52:29.206 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:28 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:52:29.206 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:28 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:52:29.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:28 smithi123 ceph-mon[24496]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:29.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:28 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:29.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:28 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:52:29.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:28 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:52:29.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:28 smithi123 ceph-mon[24496]: Adjusting osd_memory_target on smithi032 to 6434M 2024-05-03T08:52:29.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:28 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:29.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:28 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:29.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:28 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:52:29.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:28 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:52:29.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:28 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:52:30.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:29 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/3202296830' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9395950f-d4fa-4dd7-8c2b-61e5f27aabef"}]: dispatch 2024-05-03T08:52:30.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:29 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/3202296830' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9395950f-d4fa-4dd7-8c2b-61e5f27aabef"}]': finished 2024-05-03T08:52:30.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:29 smithi032 ceph-mon[22403]: osdmap e16: 3 total, 2 up, 3 in 2024-05-03T08:52:30.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:29 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-03T08:52:30.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:29 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/3202296830' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9395950f-d4fa-4dd7-8c2b-61e5f27aabef"}]: dispatch 2024-05-03T08:52:30.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:29 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/3202296830' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9395950f-d4fa-4dd7-8c2b-61e5f27aabef"}]': finished 2024-05-03T08:52:30.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:29 smithi123 ceph-mon[24496]: osdmap e16: 3 total, 2 up, 3 in 2024-05-03T08:52:30.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:29 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-03T08:52:31.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:30 smithi032 ceph-mon[22403]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:31.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:30 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/4027607319' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-03T08:52:31.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:30 smithi123 ceph-mon[24496]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:31.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:30 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/4027607319' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-03T08:52:33.155 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:32 smithi032 ceph-mon[22403]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:33.155 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:33.155 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:33.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:32 smithi123 ceph-mon[24496]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:33.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:33.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:35.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:34 smithi032 ceph-mon[22403]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:35.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:34 smithi123 ceph-mon[24496]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:36.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:36 smithi032 ceph-mon[22403]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:36.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:36 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-03T08:52:36.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:36 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:52:36.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:36 smithi032 ceph-mon[22403]: Deploying daemon osd.2 on smithi032 2024-05-03T08:52:37.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:36 smithi123 ceph-mon[24496]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:37.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:36 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-03T08:52:37.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:36 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:52:37.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:36 smithi123 ceph-mon[24496]: Deploying daemon osd.2 on smithi032 2024-05-03T08:52:38.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:38 smithi032 ceph-mon[22403]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:39.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:38 smithi123 ceph-mon[24496]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:40.858 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:40 smithi032 ceph-mon[22403]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:41.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:40 smithi123 ceph-mon[24496]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:42.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:42.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:52:42.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:52:42.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:52:42.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:52:42.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:52:42.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:42.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:52:42.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:52:42.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:52:42.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:52:42.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:52:43.172 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:43 smithi032 ceph-mon[22403]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:43.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:43 smithi123 ceph-mon[24496]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:44.958 INFO:teuthology.orchestra.run.smithi032.stdout:Created osd(s) 2 on host 'smithi032' 2024-05-03T08:52:45.252 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:45 smithi032 ceph-mon[22403]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:45.252 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:45 smithi032 ceph-mon[22403]: from='osd.2 [v2:172.21.15.32:6818/3583854825,v1:172.21.15.32:6819/3583854825]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-03T08:52:45.253 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:45 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:45.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:45 smithi123 ceph-mon[24496]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:45.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:45 smithi123 ceph-mon[24496]: from='osd.2 [v2:172.21.15.32:6818/3583854825,v1:172.21.15.32:6819/3583854825]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-03T08:52:45.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:45 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:45.553 DEBUG:teuthology.orchestra.run.smithi032:osd.2> sudo journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.2.service 2024-05-03T08:52:45.556 INFO:tasks.cephadm:Deploying osd.3 on smithi032 with /dev/vg_nvme/lv_1... 2024-05-03T08:52:45.556 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-05-03T08:52:46.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:46 smithi032 ceph-mon[22403]: from='osd.2 [v2:172.21.15.32:6818/3583854825,v1:172.21.15.32:6819/3583854825]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-03T08:52:46.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:46 smithi032 ceph-mon[22403]: osdmap e17: 3 total, 2 up, 3 in 2024-05-03T08:52:46.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:46 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-03T08:52:46.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:46 smithi032 ceph-mon[22403]: from='osd.2 [v2:172.21.15.32:6818/3583854825,v1:172.21.15.32:6819/3583854825]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-05-03T08:52:46.388 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:52:46 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-2[40608]: 2024-05-03T08:52:46.130+0000 7f1d3ed4c700 -1 osd.2 0 waiting for initial osdmap 2024-05-03T08:52:46.388 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:52:46 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-2[40608]: 2024-05-03T08:52:46.134+0000 7f1d3b6e7700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-03T08:52:46.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:46 smithi123 ceph-mon[24496]: from='osd.2 [v2:172.21.15.32:6818/3583854825,v1:172.21.15.32:6819/3583854825]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-03T08:52:46.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:46 smithi123 ceph-mon[24496]: osdmap e17: 3 total, 2 up, 3 in 2024-05-03T08:52:46.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:46 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-03T08:52:46.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:46 smithi123 ceph-mon[24496]: from='osd.2 [v2:172.21.15.32:6818/3583854825,v1:172.21.15.32:6819/3583854825]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-05-03T08:52:47.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:47 smithi032 ceph-mon[22403]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:47.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:47 smithi032 ceph-mon[22403]: from='osd.2 [v2:172.21.15.32:6818/3583854825,v1:172.21.15.32:6819/3583854825]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-05-03T08:52:47.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:47 smithi032 ceph-mon[22403]: osdmap e18: 3 total, 2 up, 3 in 2024-05-03T08:52:47.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:47 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-03T08:52:47.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:47 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-03T08:52:47.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:47 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-03T08:52:47.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:47 smithi123 ceph-mon[24496]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:47.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:47 smithi123 ceph-mon[24496]: from='osd.2 [v2:172.21.15.32:6818/3583854825,v1:172.21.15.32:6819/3583854825]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-05-03T08:52:47.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:47 smithi123 ceph-mon[24496]: osdmap e18: 3 total, 2 up, 3 in 2024-05-03T08:52:47.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:47 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-03T08:52:47.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:47 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-03T08:52:47.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:47 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-03T08:52:48.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:48 smithi032 ceph-mon[22403]: purged_snaps scrub starts 2024-05-03T08:52:48.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:48 smithi032 ceph-mon[22403]: purged_snaps scrub ok 2024-05-03T08:52:48.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:48 smithi032 ceph-mon[22403]: osd.2 [v2:172.21.15.32:6818/3583854825,v1:172.21.15.32:6819/3583854825] boot 2024-05-03T08:52:48.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:48 smithi032 ceph-mon[22403]: osdmap e19: 3 total, 3 up, 3 in 2024-05-03T08:52:48.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:48 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-03T08:52:48.391 INFO:teuthology.orchestra.run.smithi032.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-05-03T08:52:48.392 INFO:teuthology.orchestra.run.smithi032.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-05-03T08:52:48.392 INFO:teuthology.orchestra.run.smithi032.stderr: stderr: 10+0 records in 2024-05-03T08:52:48.392 INFO:teuthology.orchestra.run.smithi032.stderr:10+0 records out 2024-05-03T08:52:48.392 INFO:teuthology.orchestra.run.smithi032.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0106195 s, 987 MB/s 2024-05-03T08:52:48.392 INFO:teuthology.orchestra.run.smithi032.stderr:--> Zapping successful for: 2024-05-03T08:52:48.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:48 smithi123 ceph-mon[24496]: purged_snaps scrub starts 2024-05-03T08:52:48.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:48 smithi123 ceph-mon[24496]: purged_snaps scrub ok 2024-05-03T08:52:48.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:48 smithi123 ceph-mon[24496]: osd.2 [v2:172.21.15.32:6818/3583854825,v1:172.21.15.32:6819/3583854825] boot 2024-05-03T08:52:48.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:48 smithi123 ceph-mon[24496]: osdmap e19: 3 total, 3 up, 3 in 2024-05-03T08:52:48.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:48 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-03T08:52:49.014 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi032:vg_nvme/lv_1 2024-05-03T08:52:49.495 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:49 smithi032 ceph-mon[22403]: pgmap v70: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:49.495 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-05-03T08:52:49.495 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-05-03T08:52:49.496 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:49 smithi032 ceph-mon[22403]: osdmap e20: 3 total, 3 up, 3 in 2024-05-03T08:52:49.496 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-03T08:52:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:49 smithi123 ceph-mon[24496]: pgmap v70: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-05-03T08:52:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-05-03T08:52:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-05-03T08:52:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:49 smithi123 ceph-mon[24496]: osdmap e20: 3 total, 3 up, 3 in 2024-05-03T08:52:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-03T08:52:50.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:50 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-03T08:52:50.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:50 smithi032 ceph-mon[22403]: osdmap e21: 3 total, 3 up, 3 in 2024-05-03T08:52:50.388 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:52:50 smithi032 sudo[41987]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-03T08:52:50.388 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:52:50 smithi032 sudo[41987]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-03T08:52:50.388 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:52:50 smithi032 sudo[41987]: pam_unix(sudo:session): session closed for user root 2024-05-03T08:52:50.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:50 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-03T08:52:50.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:50 smithi123 ceph-mon[24496]: osdmap e21: 3 total, 3 up, 3 in 2024-05-03T08:52:50.884 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:52:50 smithi032 sudo[42069]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-03T08:52:50.884 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:52:50 smithi032 sudo[42069]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-03T08:52:50.884 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:52:50 smithi032 sudo[42069]: pam_unix(sudo:session): session closed for user root 2024-05-03T08:52:51.405 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:52:51 smithi032 sudo[42117]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-03T08:52:51.405 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:52:51 smithi032 sudo[42117]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-03T08:52:51.405 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:52:51 smithi032 sudo[42117]: pam_unix(sudo:session): session closed for user root 2024-05-03T08:52:51.406 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:51 smithi032 ceph-mon[22403]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-03T08:52:51.406 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:51 smithi032 ceph-mon[22403]: osdmap e22: 3 total, 3 up, 3 in 2024-05-03T08:52:51.406 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:51 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:51.406 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:51 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:51.659 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:52:51 smithi032 sudo[42160]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-03T08:52:51.659 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:52:51 smithi032 sudo[42160]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-03T08:52:51.660 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:52:51 smithi032 sudo[42160]: pam_unix(sudo:session): session closed for user root 2024-05-03T08:52:51.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:51 smithi123 ceph-mon[24496]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-03T08:52:51.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:51 smithi123 ceph-mon[24496]: osdmap e22: 3 total, 3 up, 3 in 2024-05-03T08:52:51.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:51 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:51.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:51 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:52.353 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:52:52 smithi032 sudo[42230]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-03T08:52:52.353 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:52:52 smithi032 sudo[42230]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-03T08:52:52.353 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:52:52 smithi032 sudo[42230]: pam_unix(sudo:session): session closed for user root 2024-05-03T08:52:52.354 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:52 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-03T08:52:52.354 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:52 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-03T08:52:52.354 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:52 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:52:52.354 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:52 smithi032 ceph-mon[22403]: mgrmap e19: smithi032.umllay(active, since 2m), standbys: smithi123.xrxinh 2024-05-03T08:52:52.638 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:52:52 smithi032 sudo[42266]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-03T08:52:52.638 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:52:52 smithi032 sudo[42266]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-03T08:52:52.638 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:52:52 smithi032 sudo[42266]: pam_unix(sudo:session): session closed for user root 2024-05-03T08:52:52.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:52 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-03T08:52:52.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:52 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-03T08:52:52.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:52 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:52:52.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:52 smithi123 ceph-mon[24496]: mgrmap e19: smithi032.umllay(active, since 2m), standbys: smithi123.xrxinh 2024-05-03T08:52:53.267 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:52 smithi032 sudo[42293]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-03T08:52:53.268 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:52 smithi032 sudo[42293]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-03T08:52:53.268 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:52 smithi032 sudo[42293]: pam_unix(sudo:session): session closed for user root 2024-05-03T08:52:53.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:53 smithi032 sudo[42319]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-03T08:52:53.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:53 smithi032 sudo[42319]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-03T08:52:53.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:53 smithi032 sudo[42319]: pam_unix(sudo:session): session closed for user root 2024-05-03T08:52:53.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:53 smithi032 ceph-mon[22403]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:52:53.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:53 smithi032 ceph-mon[22403]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-03T08:52:53.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:53 smithi032 ceph-mon[22403]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-03T08:52:53.699 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:53 smithi123 ceph-mon[24496]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi032:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:52:53.699 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:53 smithi123 ceph-mon[24496]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-03T08:52:53.699 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:53 smithi123 ceph-mon[24496]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-03T08:52:53.699 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:53 smithi123 sudo[26798]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-03T08:52:53.699 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:53 smithi123 sudo[26798]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-03T08:52:53.700 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:53 smithi123 sudo[26798]: pam_unix(sudo:session): session closed for user root 2024-05-03T08:52:53.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:53 smithi123 sudo[26801]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-03T08:52:53.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:53 smithi123 sudo[26801]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-03T08:52:53.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:53 smithi123 sudo[26801]: pam_unix(sudo:session): session closed for user root 2024-05-03T08:52:54.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:54 smithi123 ceph-mon[24496]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-03T08:52:54.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:54 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi032"}]: dispatch 2024-05-03T08:52:54.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:54 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-05-03T08:52:54.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:54 smithi123 ceph-mon[24496]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-03T08:52:54.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:54 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi032"}]: dispatch 2024-05-03T08:52:54.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:54 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-05-03T08:52:54.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:54 smithi123 ceph-mon[24496]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-03T08:52:54.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:54 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:54.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:54 smithi032 ceph-mon[22403]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-03T08:52:54.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:54 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi032"}]: dispatch 2024-05-03T08:52:54.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:54 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-05-03T08:52:54.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:54 smithi032 ceph-mon[22403]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-03T08:52:54.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:54 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi032"}]: dispatch 2024-05-03T08:52:54.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:54 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-05-03T08:52:54.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:54 smithi032 ceph-mon[22403]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-03T08:52:54.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:54 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:55.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:55 smithi032 ceph-mon[22403]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-05-03T08:52:55.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:55 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:55.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:55 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:52:55.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:55 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:52:55.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:55 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:52:55.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:55 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:55.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:55 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:55.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:55 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:52:55.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:55 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:52:55.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:55 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:52:55.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:55 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:55.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:55 smithi123 ceph-mon[24496]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-05-03T08:52:55.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:55 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:55.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:55 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:52:55.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:55 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:52:55.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:55 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:52:55.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:55 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:55.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:55 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:55.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:55 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:52:55.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:55 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:52:55.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:55 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:52:55.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:55 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:52:56.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:56 smithi123 ceph-mon[24496]: Detected new or changed devices on smithi032 2024-05-03T08:52:56.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:56 smithi123 ceph-mon[24496]: Adjusting osd_memory_target on smithi032 to 4289M 2024-05-03T08:52:56.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:56 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/4076263433' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e68bbfc1-e00a-46fd-a30d-8047f56633e3"}]: dispatch 2024-05-03T08:52:56.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:56 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/4076263433' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e68bbfc1-e00a-46fd-a30d-8047f56633e3"}]': finished 2024-05-03T08:52:56.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:56 smithi123 ceph-mon[24496]: osdmap e23: 4 total, 3 up, 4 in 2024-05-03T08:52:56.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:56 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-03T08:52:56.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:56 smithi032 ceph-mon[22403]: Detected new or changed devices on smithi032 2024-05-03T08:52:56.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:56 smithi032 ceph-mon[22403]: Adjusting osd_memory_target on smithi032 to 4289M 2024-05-03T08:52:56.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:56 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/4076263433' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e68bbfc1-e00a-46fd-a30d-8047f56633e3"}]: dispatch 2024-05-03T08:52:56.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:56 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/4076263433' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e68bbfc1-e00a-46fd-a30d-8047f56633e3"}]': finished 2024-05-03T08:52:56.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:56 smithi032 ceph-mon[22403]: osdmap e23: 4 total, 3 up, 4 in 2024-05-03T08:52:56.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:56 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-03T08:52:57.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:57 smithi123 ceph-mon[24496]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-05-03T08:52:57.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:57 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/4199882407' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-03T08:52:57.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:57 smithi032 ceph-mon[22403]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-05-03T08:52:57.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:57 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/4199882407' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-03T08:52:59.568 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:52:59 smithi032 ceph-mon[22403]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-05-03T08:52:59.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:52:59 smithi123 ceph-mon[24496]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-05-03T08:53:01.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:01 smithi032 ceph-mon[22403]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-05-03T08:53:01.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:01 smithi123 ceph-mon[24496]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-05-03T08:53:03.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:03 smithi123 ceph-mon[24496]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-05-03T08:53:03.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:03 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-03T08:53:03.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:03 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:03.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:03 smithi123 ceph-mon[24496]: Deploying daemon osd.3 on smithi032 2024-05-03T08:53:03.851 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:03 smithi032 ceph-mon[22403]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-05-03T08:53:03.851 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:03 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-03T08:53:03.851 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:03 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:03.851 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:03 smithi032 ceph-mon[22403]: Deploying daemon osd.3 on smithi032 2024-05-03T08:53:05.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:05 smithi123 ceph-mon[24496]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-05-03T08:53:05.774 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:05 smithi032 ceph-mon[22403]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-05-03T08:53:07.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:07 smithi123 ceph-mon[24496]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-05-03T08:53:07.746 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:07 smithi032 ceph-mon[22403]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-05-03T08:53:09.135 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:08 smithi032 ceph-mon[22403]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 67 KiB/s wr, 4 op/s 2024-05-03T08:53:09.135 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:08 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:09.136 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:08 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:53:09.136 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:08 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:09.136 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:08 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:53:09.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:08 smithi123 ceph-mon[24496]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 67 KiB/s wr, 4 op/s 2024-05-03T08:53:09.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:08 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:09.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:08 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:53:09.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:08 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:09.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:08 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:53:10.833 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:10 smithi032 ceph-mon[22403]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-03T08:53:11.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:10 smithi123 ceph-mon[24496]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-03T08:53:11.282 INFO:teuthology.orchestra.run.smithi032.stdout:Created osd(s) 3 on host 'smithi032' 2024-05-03T08:53:11.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:11 smithi032 ceph-mon[22403]: from='osd.3 [v2:172.21.15.32:6826/3336131837,v1:172.21.15.32:6827/3336131837]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-03T08:53:11.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:11 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:12.163 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:11 smithi123 ceph-mon[24496]: from='osd.3 [v2:172.21.15.32:6826/3336131837,v1:172.21.15.32:6827/3336131837]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-03T08:53:12.163 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:11 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:12.279 DEBUG:teuthology.orchestra.run.smithi032:osd.3> sudo journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.3.service 2024-05-03T08:53:12.281 INFO:tasks.cephadm:Deploying osd.4 on smithi123 with /dev/vg_nvme/lv_4... 2024-05-03T08:53:12.281 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-05-03T08:53:12.837 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:12 smithi123 ceph-mon[24496]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-03T08:53:12.837 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:12 smithi123 ceph-mon[24496]: from='osd.3 [v2:172.21.15.32:6826/3336131837,v1:172.21.15.32:6827/3336131837]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-03T08:53:12.837 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:12 smithi123 ceph-mon[24496]: osdmap e24: 4 total, 3 up, 4 in 2024-05-03T08:53:12.838 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:12 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-03T08:53:12.838 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:12 smithi123 ceph-mon[24496]: from='osd.3 [v2:172.21.15.32:6826/3336131837,v1:172.21.15.32:6827/3336131837]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-05-03T08:53:12.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:12 smithi032 ceph-mon[22403]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-03T08:53:12.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:12 smithi032 ceph-mon[22403]: from='osd.3 [v2:172.21.15.32:6826/3336131837,v1:172.21.15.32:6827/3336131837]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-03T08:53:12.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:12 smithi032 ceph-mon[22403]: osdmap e24: 4 total, 3 up, 4 in 2024-05-03T08:53:12.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:12 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-03T08:53:12.889 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:12 smithi032 ceph-mon[22403]: from='osd.3 [v2:172.21.15.32:6826/3336131837,v1:172.21.15.32:6827/3336131837]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]: dispatch 2024-05-03T08:53:12.889 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:53:12 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-3[43892]: 2024-05-03T08:53:12.767+0000 7f511a262700 -1 osd.3 0 waiting for initial osdmap 2024-05-03T08:53:12.889 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:53:12 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-3[43892]: 2024-05-03T08:53:12.772+0000 7f51143f8700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-03T08:53:13.866 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:13 smithi123 ceph-mon[24496]: from='osd.3 [v2:172.21.15.32:6826/3336131837,v1:172.21.15.32:6827/3336131837]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-05-03T08:53:13.867 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:13 smithi123 ceph-mon[24496]: osdmap e25: 4 total, 3 up, 4 in 2024-05-03T08:53:13.867 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:13 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-03T08:53:13.867 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:13 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-03T08:53:13.867 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:13 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-03T08:53:14.097 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:13 smithi032 ceph-mon[22403]: from='osd.3 [v2:172.21.15.32:6826/3336131837,v1:172.21.15.32:6827/3336131837]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi032", "root=default"]}]': finished 2024-05-03T08:53:14.098 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:13 smithi032 ceph-mon[22403]: osdmap e25: 4 total, 3 up, 4 in 2024-05-03T08:53:14.098 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:13 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-03T08:53:14.098 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:13 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-03T08:53:14.098 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:13 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-03T08:53:14.354 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-05-03T08:53:14.354 INFO:teuthology.orchestra.run.smithi123.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-05-03T08:53:14.354 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 10+0 records in 2024-05-03T08:53:14.354 INFO:teuthology.orchestra.run.smithi123.stderr:10+0 records out 2024-05-03T08:53:14.354 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0113886 s, 921 MB/s 2024-05-03T08:53:14.354 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping successful for: 2024-05-03T08:53:14.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:14 smithi032 ceph-mon[22403]: purged_snaps scrub starts 2024-05-03T08:53:14.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:14 smithi032 ceph-mon[22403]: purged_snaps scrub ok 2024-05-03T08:53:14.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:14 smithi032 ceph-mon[22403]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-03T08:53:14.877 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:14 smithi032 ceph-mon[22403]: osd.3 [v2:172.21.15.32:6826/3336131837,v1:172.21.15.32:6827/3336131837] boot 2024-05-03T08:53:14.878 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:14 smithi032 ceph-mon[22403]: osdmap e26: 4 total, 4 up, 4 in 2024-05-03T08:53:14.878 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:14 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-03T08:53:14.878 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:14 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:14.928 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:14 smithi123 ceph-mon[24496]: purged_snaps scrub starts 2024-05-03T08:53:14.928 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:14 smithi123 ceph-mon[24496]: purged_snaps scrub ok 2024-05-03T08:53:14.928 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:14 smithi123 ceph-mon[24496]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-03T08:53:14.929 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:14 smithi123 ceph-mon[24496]: osd.3 [v2:172.21.15.32:6826/3336131837,v1:172.21.15.32:6827/3336131837] boot 2024-05-03T08:53:14.929 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:14 smithi123 ceph-mon[24496]: osdmap e26: 4 total, 4 up, 4 in 2024-05-03T08:53:14.929 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:14 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-03T08:53:14.929 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:14 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:14.951 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi123:vg_nvme/lv_4 2024-05-03T08:53:15.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:15 smithi032 ceph-mon[22403]: osdmap e27: 4 total, 4 up, 4 in 2024-05-03T08:53:15.934 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:15 smithi123 ceph-mon[24496]: osdmap e27: 4 total, 4 up, 4 in 2024-05-03T08:53:17.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:16 smithi032 ceph-mon[22403]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:17.208 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:16 smithi123 ceph-mon[24496]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:17.963 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:17 smithi032 ceph-mon[22403]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:53:17.963 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:17 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-03T08:53:17.963 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:17 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-03T08:53:17.963 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:17 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:18.072 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:17 smithi123 ceph-mon[24496]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:53:18.072 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:17 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-03T08:53:18.073 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:17 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-03T08:53:18.073 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:17 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:19.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:18 smithi032 ceph-mon[22403]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:19.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:18 smithi032 ceph-mon[22403]: Detected new or changed devices on smithi032 2024-05-03T08:53:19.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:18 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:19.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:18 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:53:19.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:18 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:53:19.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:18 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:53:19.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:18 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:53:19.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:18 smithi032 ceph-mon[22403]: Adjusting osd_memory_target on smithi032 to 3217M 2024-05-03T08:53:19.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:18 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:19.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:18 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:19.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:18 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:53:19.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:18 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:19.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:18 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:53:19.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:18 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:19.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:18 smithi032 ceph-mon[22403]: from='client.? 172.21.15.123:0/2007712174' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84c5c1ee-8601-4517-8ac8-7312fa8a899e"}]: dispatch 2024-05-03T08:53:19.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:18 smithi032 ceph-mon[22403]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84c5c1ee-8601-4517-8ac8-7312fa8a899e"}]: dispatch 2024-05-03T08:53:19.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:18 smithi032 ceph-mon[22403]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "84c5c1ee-8601-4517-8ac8-7312fa8a899e"}]': finished 2024-05-03T08:53:19.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:18 smithi032 ceph-mon[22403]: osdmap e28: 5 total, 4 up, 5 in 2024-05-03T08:53:19.389 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:18 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-03T08:53:19.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:18 smithi123 ceph-mon[24496]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:19.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:18 smithi123 ceph-mon[24496]: Detected new or changed devices on smithi032 2024-05-03T08:53:19.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:18 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:19.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:18 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:53:19.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:18 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:53:19.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:18 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:53:19.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:18 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:53:19.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:18 smithi123 ceph-mon[24496]: Adjusting osd_memory_target on smithi032 to 3217M 2024-05-03T08:53:19.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:18 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:19.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:18 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:19.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:18 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:53:19.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:18 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:19.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:18 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:53:19.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:18 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:19.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:18 smithi123 ceph-mon[24496]: from='client.? 172.21.15.123:0/2007712174' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84c5c1ee-8601-4517-8ac8-7312fa8a899e"}]: dispatch 2024-05-03T08:53:19.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:18 smithi123 ceph-mon[24496]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84c5c1ee-8601-4517-8ac8-7312fa8a899e"}]: dispatch 2024-05-03T08:53:19.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:18 smithi123 ceph-mon[24496]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "84c5c1ee-8601-4517-8ac8-7312fa8a899e"}]': finished 2024-05-03T08:53:19.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:18 smithi123 ceph-mon[24496]: osdmap e28: 5 total, 4 up, 5 in 2024-05-03T08:53:19.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:18 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-03T08:53:20.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:20 smithi032 ceph-mon[22403]: from='client.? 172.21.15.123:0/2194634359' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-03T08:53:20.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:20 smithi123 ceph-mon[24496]: from='client.? 172.21.15.123:0/2194634359' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-03T08:53:21.263 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:21 smithi032 ceph-mon[22403]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:21.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:21 smithi123 ceph-mon[24496]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:23.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:23 smithi123 ceph-mon[24496]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:23.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:23 smithi032 ceph-mon[22403]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:25.216 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:25 smithi123 ceph-mon[24496]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:25.217 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:25 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-03T08:53:25.217 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:25 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:25.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:25 smithi032 ceph-mon[22403]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:25.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:25 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-03T08:53:25.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:25 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:26.136 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:26 smithi123 ceph-mon[24496]: Deploying daemon osd.4 on smithi123 2024-05-03T08:53:26.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:26 smithi032 ceph-mon[22403]: Deploying daemon osd.4 on smithi123 2024-05-03T08:53:27.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:27 smithi032 ceph-mon[22403]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:27.425 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:27 smithi123 ceph-mon[24496]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:29.325 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:29 smithi123 ceph-mon[24496]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:29.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:29 smithi032 ceph-mon[22403]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:30.852 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:30 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:30.852 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:30 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:53:30.852 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:30 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:30.853 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:30 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:53:30.853 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:30 smithi123 ceph-mon[24496]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:30.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:30 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:30.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:30 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:53:30.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:30 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:30.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:30 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:53:30.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:30 smithi032 ceph-mon[22403]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:32.686 INFO:teuthology.orchestra.run.smithi123.stdout:Created osd(s) 4 on host 'smithi123' 2024-05-03T08:53:32.921 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:32 smithi123 ceph-mon[24496]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:32.921 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:32 smithi123 ceph-mon[24496]: from='osd.4 [v2:172.21.15.123:6800/309107390,v1:172.21.15.123:6801/309107390]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-03T08:53:32.922 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:32 smithi123 ceph-mon[24496]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-03T08:53:32.922 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:33.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:32 smithi032 ceph-mon[22403]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:33.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:32 smithi032 ceph-mon[22403]: from='osd.4 [v2:172.21.15.123:6800/309107390,v1:172.21.15.123:6801/309107390]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-03T08:53:33.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:32 smithi032 ceph-mon[22403]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-03T08:53:33.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:33.281 DEBUG:teuthology.orchestra.run.smithi123:osd.4> sudo journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.4.service 2024-05-03T08:53:33.284 INFO:tasks.cephadm:Deploying osd.5 on smithi123 with /dev/vg_nvme/lv_3... 2024-05-03T08:53:33.284 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-05-03T08:53:33.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:33 smithi123 ceph-mon[24496]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-03T08:53:33.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:33 smithi123 ceph-mon[24496]: osdmap e29: 5 total, 4 up, 5 in 2024-05-03T08:53:33.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:33 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-03T08:53:33.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:33 smithi123 ceph-mon[24496]: from='osd.4 [v2:172.21.15.123:6800/309107390,v1:172.21.15.123:6801/309107390]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-05-03T08:53:33.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:33 smithi123 ceph-mon[24496]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-05-03T08:53:33.978 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:53:33 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4[28726]: 2024-05-03T08:53:33.767+0000 7f3108e29700 -1 osd.4 0 waiting for initial osdmap 2024-05-03T08:53:33.978 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:53:33 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4[28726]: 2024-05-03T08:53:33.773+0000 7f31057c4700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-03T08:53:34.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:33 smithi032 ceph-mon[22403]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-03T08:53:34.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:33 smithi032 ceph-mon[22403]: osdmap e29: 5 total, 4 up, 5 in 2024-05-03T08:53:34.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:33 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-03T08:53:34.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:33 smithi032 ceph-mon[22403]: from='osd.4 [v2:172.21.15.123:6800/309107390,v1:172.21.15.123:6801/309107390]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-05-03T08:53:34.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:33 smithi032 ceph-mon[22403]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-05-03T08:53:35.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:34 smithi123 ceph-mon[24496]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:35.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:34 smithi123 ceph-mon[24496]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-05-03T08:53:35.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:34 smithi123 ceph-mon[24496]: osdmap e30: 5 total, 4 up, 5 in 2024-05-03T08:53:35.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:34 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-03T08:53:35.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:34 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-03T08:53:35.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:34 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:35.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:34 smithi123 ceph-mon[24496]: osd.4 [v2:172.21.15.123:6800/309107390,v1:172.21.15.123:6801/309107390] boot 2024-05-03T08:53:35.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:34 smithi123 ceph-mon[24496]: osdmap e31: 5 total, 5 up, 5 in 2024-05-03T08:53:35.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:34 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-03T08:53:35.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:34 smithi032 ceph-mon[22403]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-03T08:53:35.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:34 smithi032 ceph-mon[22403]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-05-03T08:53:35.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:34 smithi032 ceph-mon[22403]: osdmap e30: 5 total, 4 up, 5 in 2024-05-03T08:53:35.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:34 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-03T08:53:35.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:34 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-03T08:53:35.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:34 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:35.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:34 smithi032 ceph-mon[22403]: osd.4 [v2:172.21.15.123:6800/309107390,v1:172.21.15.123:6801/309107390] boot 2024-05-03T08:53:35.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:34 smithi032 ceph-mon[22403]: osdmap e31: 5 total, 5 up, 5 in 2024-05-03T08:53:35.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:34 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-03T08:53:35.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:35 smithi032 ceph-mon[22403]: purged_snaps scrub starts 2024-05-03T08:53:35.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:35 smithi032 ceph-mon[22403]: purged_snaps scrub ok 2024-05-03T08:53:35.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:35 smithi032 ceph-mon[22403]: osdmap e32: 5 total, 5 up, 5 in 2024-05-03T08:53:36.038 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:35 smithi123 ceph-mon[24496]: purged_snaps scrub starts 2024-05-03T08:53:36.038 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:35 smithi123 ceph-mon[24496]: purged_snaps scrub ok 2024-05-03T08:53:36.038 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:35 smithi123 ceph-mon[24496]: osdmap e32: 5 total, 5 up, 5 in 2024-05-03T08:53:36.416 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-05-03T08:53:36.416 INFO:teuthology.orchestra.run.smithi123.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-05-03T08:53:36.416 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 10+0 records in 2024-05-03T08:53:36.417 INFO:teuthology.orchestra.run.smithi123.stderr:10+0 records out 2024-05-03T08:53:36.417 INFO:teuthology.orchestra.run.smithi123.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0122374 s, 857 MB/s 2024-05-03T08:53:36.417 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping successful for: 2024-05-03T08:53:37.052 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:36 smithi123 ceph-mon[24496]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-03T08:53:37.052 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:36 smithi123 ceph-mon[24496]: osdmap e33: 5 total, 5 up, 5 in 2024-05-03T08:53:37.075 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi123:vg_nvme/lv_3 2024-05-03T08:53:37.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:36 smithi032 ceph-mon[22403]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-03T08:53:37.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:36 smithi032 ceph-mon[22403]: osdmap e33: 5 total, 5 up, 5 in 2024-05-03T08:53:38.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:38 smithi123 ceph-mon[24496]: pgmap v109: 1 pgs: 1 active+recovering+undersized+degraded+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 2/6 objects misplaced (33.333%) 2024-05-03T08:53:38.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:38 smithi123 ceph-mon[24496]: osdmap e34: 5 total, 5 up, 5 in 2024-05-03T08:53:38.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:38 smithi123 ceph-mon[24496]: Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2024-05-03T08:53:39.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:38 smithi032 ceph-mon[22403]: pgmap v109: 1 pgs: 1 active+recovering+undersized+degraded+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 2/6 objects misplaced (33.333%) 2024-05-03T08:53:39.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:38 smithi032 ceph-mon[22403]: osdmap e34: 5 total, 5 up, 5 in 2024-05-03T08:53:39.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:38 smithi032 ceph-mon[22403]: Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2024-05-03T08:53:39.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:39 smithi123 ceph-mon[24496]: osdmap e35: 5 total, 5 up, 5 in 2024-05-03T08:53:39.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:39 smithi123 ceph-mon[24496]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:53:39.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:39 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-03T08:53:39.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:39 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-03T08:53:39.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:39 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:39.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:39 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:39.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:39 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:53:39.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:39 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:39.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:39 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:39.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:39 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:53:39.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:39 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:39.823 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:39 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:53:39.823 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:39 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:40.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:39 smithi032 ceph-mon[22403]: osdmap e35: 5 total, 5 up, 5 in 2024-05-03T08:53:40.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:39 smithi032 ceph-mon[22403]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:53:40.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:39 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-03T08:53:40.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:39 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-03T08:53:40.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:39 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:40.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:39 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:40.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:39 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:53:40.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:39 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:40.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:39 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:40.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:39 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:53:40.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:39 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:40.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:39 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:53:40.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:39 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:40.818 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:40 smithi123 ceph-mon[24496]: Detected new or changed devices on smithi123 2024-05-03T08:53:40.818 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:40 smithi123 ceph-mon[24496]: Adjusting osd_memory_target on smithi123 to 15936M 2024-05-03T08:53:40.818 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:40 smithi123 ceph-mon[24496]: pgmap v112: 1 pgs: 1 active+recovering+undersized+degraded+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 2/6 objects misplaced (33.333%) 2024-05-03T08:53:41.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:40 smithi032 ceph-mon[22403]: Detected new or changed devices on smithi123 2024-05-03T08:53:41.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:40 smithi032 ceph-mon[22403]: Adjusting osd_memory_target on smithi123 to 15936M 2024-05-03T08:53:41.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:40 smithi032 ceph-mon[22403]: pgmap v112: 1 pgs: 1 active+recovering+undersized+degraded+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 2/6 objects misplaced (33.333%) 2024-05-03T08:53:42.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:41 smithi032 ceph-mon[22403]: from='client.? 172.21.15.123:0/2386514643' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6cdcf768-83e1-48a6-88d5-2b01d7efb20c"}]: dispatch 2024-05-03T08:53:42.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:41 smithi032 ceph-mon[22403]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6cdcf768-83e1-48a6-88d5-2b01d7efb20c"}]: dispatch 2024-05-03T08:53:42.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:41 smithi032 ceph-mon[22403]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6cdcf768-83e1-48a6-88d5-2b01d7efb20c"}]': finished 2024-05-03T08:53:42.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:41 smithi032 ceph-mon[22403]: osdmap e36: 6 total, 5 up, 6 in 2024-05-03T08:53:42.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:41 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-03T08:53:42.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:41 smithi123 ceph-mon[24496]: from='client.? 172.21.15.123:0/2386514643' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6cdcf768-83e1-48a6-88d5-2b01d7efb20c"}]: dispatch 2024-05-03T08:53:42.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:41 smithi123 ceph-mon[24496]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6cdcf768-83e1-48a6-88d5-2b01d7efb20c"}]: dispatch 2024-05-03T08:53:42.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:41 smithi123 ceph-mon[24496]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6cdcf768-83e1-48a6-88d5-2b01d7efb20c"}]': finished 2024-05-03T08:53:42.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:41 smithi123 ceph-mon[24496]: osdmap e36: 6 total, 5 up, 6 in 2024-05-03T08:53:42.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:41 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-03T08:53:43.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:42 smithi032 ceph-mon[22403]: pgmap v114: 1 pgs: 1 active+recovering+undersized+degraded+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 2/6 objects misplaced (33.333%) 2024-05-03T08:53:43.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:43.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:53:43.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:53:43.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:42 smithi032 ceph-mon[22403]: from='client.? 172.21.15.123:0/4044348882' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-03T08:53:43.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:42 smithi123 ceph-mon[24496]: pgmap v114: 1 pgs: 1 active+recovering+undersized+degraded+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 2/6 objects misplaced (33.333%) 2024-05-03T08:53:43.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:43.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:53:43.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:53:43.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:42 smithi123 ceph-mon[24496]: from='client.? 172.21.15.123:0/4044348882' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-03T08:53:44.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:43 smithi032 ceph-mon[22403]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2024-05-03T08:53:44.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:43 smithi032 ceph-mon[22403]: Cluster is now healthy 2024-05-03T08:53:44.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:43 smithi123 ceph-mon[24496]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2024-05-03T08:53:44.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:43 smithi123 ceph-mon[24496]: Cluster is now healthy 2024-05-03T08:53:44.946 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:44 smithi123 ceph-mon[24496]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-03T08:53:45.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:44 smithi032 ceph-mon[22403]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-03T08:53:47.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:46 smithi032 ceph-mon[22403]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-03T08:53:47.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:46 smithi123 ceph-mon[24496]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-03T08:53:48.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:47 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-03T08:53:48.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:47 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:48.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:47 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-03T08:53:48.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:47 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:48.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:48 smithi123 ceph-mon[24496]: Deploying daemon osd.5 on smithi123 2024-05-03T08:53:48.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:48 smithi123 ceph-mon[24496]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-05-03T08:53:49.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:48 smithi032 ceph-mon[22403]: Deploying daemon osd.5 on smithi123 2024-05-03T08:53:49.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:48 smithi032 ceph-mon[22403]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-05-03T08:53:51.045 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:50 smithi123 ceph-mon[24496]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-03T08:53:51.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:50 smithi032 ceph-mon[22403]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-03T08:53:52.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:52 smithi123 ceph-mon[24496]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-05-03T08:53:52.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:52 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:52.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:52 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:53:52.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:52 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:52.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:52 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:53:53.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:52 smithi032 ceph-mon[22403]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-05-03T08:53:53.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:52 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:53.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:52 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:53:53.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:52 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:53:53.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:52 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:53:54.901 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:54 smithi032 ceph-mon[22403]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-05-03T08:53:54.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:54 smithi123 ceph-mon[24496]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-05-03T08:53:55.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:55 smithi123 ceph-mon[24496]: from='osd.5 [v2:172.21.15.123:6808/1030990683,v1:172.21.15.123:6809/1030990683]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-03T08:53:55.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:55 smithi123 ceph-mon[24496]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-03T08:53:56.104 INFO:teuthology.orchestra.run.smithi123.stdout:Created osd(s) 5 on host 'smithi123' 2024-05-03T08:53:56.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:55 smithi032 ceph-mon[22403]: from='osd.5 [v2:172.21.15.123:6808/1030990683,v1:172.21.15.123:6809/1030990683]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-03T08:53:56.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:55 smithi032 ceph-mon[22403]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-03T08:53:56.726 DEBUG:teuthology.orchestra.run.smithi123:osd.5> sudo journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.5.service 2024-05-03T08:53:56.728 INFO:tasks.cephadm:Deploying osd.6 on smithi123 with /dev/vg_nvme/lv_2... 2024-05-03T08:53:56.728 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-05-03T08:53:56.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:56 smithi123 ceph-mon[24496]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-03T08:53:56.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:56 smithi123 ceph-mon[24496]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-03T08:53:56.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:56 smithi123 ceph-mon[24496]: osdmap e37: 6 total, 5 up, 6 in 2024-05-03T08:53:56.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:56 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-03T08:53:56.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:56 smithi123 ceph-mon[24496]: from='osd.5 [v2:172.21.15.123:6808/1030990683,v1:172.21.15.123:6809/1030990683]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-05-03T08:53:56.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:56 smithi123 ceph-mon[24496]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-05-03T08:53:56.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:56 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:56.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:56 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:56.978 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:53:56 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:53:56.853+0000 7f3c582b2700 -1 osd.5 0 waiting for initial osdmap 2024-05-03T08:53:56.978 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:53:56 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:53:56.858+0000 7f3c53c4b700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-03T08:53:57.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:56 smithi032 ceph-mon[22403]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-03T08:53:57.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:56 smithi032 ceph-mon[22403]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-03T08:53:57.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:56 smithi032 ceph-mon[22403]: osdmap e37: 6 total, 5 up, 6 in 2024-05-03T08:53:57.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:56 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-03T08:53:57.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:56 smithi032 ceph-mon[22403]: from='osd.5 [v2:172.21.15.123:6808/1030990683,v1:172.21.15.123:6809/1030990683]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-05-03T08:53:57.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:56 smithi032 ceph-mon[22403]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-05-03T08:53:57.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:56 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:57.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:56 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:58.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:57 smithi032 ceph-mon[22403]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-05-03T08:53:58.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:57 smithi032 ceph-mon[22403]: osdmap e38: 6 total, 5 up, 6 in 2024-05-03T08:53:58.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:57 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-03T08:53:58.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:57 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-03T08:53:58.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:57 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:58.216 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:57 smithi123 ceph-mon[24496]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-05-03T08:53:58.216 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:57 smithi123 ceph-mon[24496]: osdmap e38: 6 total, 5 up, 6 in 2024-05-03T08:53:58.216 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:57 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-03T08:53:58.216 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:57 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-03T08:53:58.216 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:57 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:53:58.998 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-05-03T08:53:58.998 INFO:teuthology.orchestra.run.smithi123.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-05-03T08:53:58.998 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 10+0 records in 2024-05-03T08:53:58.998 INFO:teuthology.orchestra.run.smithi123.stderr:10+0 records out 2024-05-03T08:53:58.998 INFO:teuthology.orchestra.run.smithi123.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0125029 s, 839 MB/s 2024-05-03T08:53:58.998 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping successful for: 2024-05-03T08:53:59.074 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:58 smithi123 ceph-mon[24496]: purged_snaps scrub starts 2024-05-03T08:53:59.074 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:58 smithi123 ceph-mon[24496]: purged_snaps scrub ok 2024-05-03T08:53:59.074 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:58 smithi123 ceph-mon[24496]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-03T08:53:59.074 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:58 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-03T08:53:59.074 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:58 smithi123 ceph-mon[24496]: osd.5 [v2:172.21.15.123:6808/1030990683,v1:172.21.15.123:6809/1030990683] boot 2024-05-03T08:53:59.074 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:58 smithi123 ceph-mon[24496]: osdmap e39: 6 total, 6 up, 6 in 2024-05-03T08:53:59.074 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:58 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-03T08:53:59.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:58 smithi032 ceph-mon[22403]: purged_snaps scrub starts 2024-05-03T08:53:59.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:58 smithi032 ceph-mon[22403]: purged_snaps scrub ok 2024-05-03T08:53:59.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:58 smithi032 ceph-mon[22403]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-03T08:53:59.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:58 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-03T08:53:59.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:58 smithi032 ceph-mon[22403]: osd.5 [v2:172.21.15.123:6808/1030990683,v1:172.21.15.123:6809/1030990683] boot 2024-05-03T08:53:59.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:58 smithi032 ceph-mon[22403]: osdmap e39: 6 total, 6 up, 6 in 2024-05-03T08:53:59.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:58 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-03T08:53:59.667 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi123:vg_nvme/lv_2 2024-05-03T08:53:59.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:53:59 smithi123 ceph-mon[24496]: osdmap e40: 6 total, 6 up, 6 in 2024-05-03T08:54:00.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:53:59 smithi032 ceph-mon[22403]: osdmap e40: 6 total, 6 up, 6 in 2024-05-03T08:54:00.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:00 smithi123 ceph-mon[24496]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-03T08:54:00.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:00 smithi123 ceph-mon[24496]: osdmap e41: 6 total, 6 up, 6 in 2024-05-03T08:54:01.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:00 smithi032 ceph-mon[22403]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-03T08:54:01.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:00 smithi032 ceph-mon[22403]: osdmap e41: 6 total, 6 up, 6 in 2024-05-03T08:54:02.788 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:02 smithi123 ceph-mon[24496]: pgmap v129: 1 pgs: 1 active+recovering+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 1/6 objects misplaced (16.667%) 2024-05-03T08:54:02.788 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:02 smithi123 ceph-mon[24496]: osdmap e42: 6 total, 6 up, 6 in 2024-05-03T08:54:02.788 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:02 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:02.788 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:02 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:54:02.788 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:02 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:54:02.789 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:02 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:02.789 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:02 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:02.789 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:02 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:54:02.789 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:02 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:54:02.789 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:02 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:54:02.789 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:02 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:03.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:02 smithi032 ceph-mon[22403]: pgmap v129: 1 pgs: 1 active+recovering+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 1/6 objects misplaced (16.667%) 2024-05-03T08:54:03.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:02 smithi032 ceph-mon[22403]: osdmap e42: 6 total, 6 up, 6 in 2024-05-03T08:54:03.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:02 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:03.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:02 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:54:03.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:02 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:54:03.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:02 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:03.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:02 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:03.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:02 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:54:03.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:02 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:54:03.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:02 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:54:03.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:02 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:03.891 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:03 smithi123 ceph-mon[24496]: Detected new or changed devices on smithi123 2024-05-03T08:54:03.891 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:03 smithi123 ceph-mon[24496]: Adjusting osd_memory_target on smithi123 to 7968M 2024-05-03T08:54:03.892 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:03 smithi123 ceph-mon[24496]: osdmap e43: 6 total, 6 up, 6 in 2024-05-03T08:54:03.892 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:03 smithi123 ceph-mon[24496]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:54:03.892 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:03 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-03T08:54:03.892 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:03 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-03T08:54:03.892 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:03 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:54:04.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:03 smithi032 ceph-mon[22403]: Detected new or changed devices on smithi123 2024-05-03T08:54:04.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:03 smithi032 ceph-mon[22403]: Adjusting osd_memory_target on smithi123 to 7968M 2024-05-03T08:54:04.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:03 smithi032 ceph-mon[22403]: osdmap e43: 6 total, 6 up, 6 in 2024-05-03T08:54:04.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:03 smithi032 ceph-mon[22403]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:54:04.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:03 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-03T08:54:04.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:03 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-03T08:54:04.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:03 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:54:04.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:04 smithi123 ceph-mon[24496]: pgmap v132: 1 pgs: 1 active+recovering+undersized+remapped; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 1/6 objects misplaced (16.667%) 2024-05-03T08:54:05.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:04 smithi032 ceph-mon[22403]: pgmap v132: 1 pgs: 1 active+recovering+undersized+remapped; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 1/6 objects misplaced (16.667%) 2024-05-03T08:54:06.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:05 smithi032 ceph-mon[22403]: from='client.? 172.21.15.123:0/3318206964' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "526d3fa7-1ba8-4921-8caa-86b6f9643543"}]: dispatch 2024-05-03T08:54:06.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:05 smithi032 ceph-mon[22403]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "526d3fa7-1ba8-4921-8caa-86b6f9643543"}]: dispatch 2024-05-03T08:54:06.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:05 smithi032 ceph-mon[22403]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "526d3fa7-1ba8-4921-8caa-86b6f9643543"}]': finished 2024-05-03T08:54:06.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:05 smithi032 ceph-mon[22403]: osdmap e44: 7 total, 6 up, 7 in 2024-05-03T08:54:06.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:05 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-03T08:54:06.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:05 smithi123 ceph-mon[24496]: from='client.? 172.21.15.123:0/3318206964' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "526d3fa7-1ba8-4921-8caa-86b6f9643543"}]: dispatch 2024-05-03T08:54:06.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:05 smithi123 ceph-mon[24496]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "526d3fa7-1ba8-4921-8caa-86b6f9643543"}]: dispatch 2024-05-03T08:54:06.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:05 smithi123 ceph-mon[24496]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "526d3fa7-1ba8-4921-8caa-86b6f9643543"}]': finished 2024-05-03T08:54:06.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:05 smithi123 ceph-mon[24496]: osdmap e44: 7 total, 6 up, 7 in 2024-05-03T08:54:06.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:05 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-03T08:54:07.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:06 smithi032 ceph-mon[22403]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-03T08:54:07.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:06 smithi032 ceph-mon[22403]: from='client.? 172.21.15.123:0/1676523430' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-03T08:54:07.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:06 smithi123 ceph-mon[24496]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-03T08:54:07.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:06 smithi123 ceph-mon[24496]: from='client.? 172.21.15.123:0/1676523430' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-03T08:54:08.861 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:08 smithi123 ceph-mon[24496]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-05-03T08:54:09.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:08 smithi032 ceph-mon[22403]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-05-03T08:54:11.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:10 smithi032 ceph-mon[22403]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 40 KiB/s, 0 objects/s recovering 2024-05-03T08:54:11.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:10 smithi123 ceph-mon[24496]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 40 KiB/s, 0 objects/s recovering 2024-05-03T08:54:11.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:11 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-03T08:54:11.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:11 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:54:12.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:11 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-03T08:54:12.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:11 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:54:12.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:12 smithi123 ceph-mon[24496]: Deploying daemon osd.6 on smithi123 2024-05-03T08:54:12.973 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:12 smithi123 ceph-mon[24496]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 36 KiB/s, 0 objects/s recovering 2024-05-03T08:54:13.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:12 smithi032 ceph-mon[22403]: Deploying daemon osd.6 on smithi123 2024-05-03T08:54:13.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:12 smithi032 ceph-mon[22403]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 36 KiB/s, 0 objects/s recovering 2024-05-03T08:54:15.108 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:14 smithi123 ceph-mon[24496]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 32 KiB/s, 0 objects/s recovering 2024-05-03T08:54:15.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:14 smithi032 ceph-mon[22403]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 32 KiB/s, 0 objects/s recovering 2024-05-03T08:54:16.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:16 smithi123 ceph-mon[24496]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-03T08:54:16.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:16 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:16.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:16 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:54:16.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:16 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:54:16.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:16 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:54:17.059 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:16 smithi032 ceph-mon[22403]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-03T08:54:17.059 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:16 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:17.059 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:16 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:54:17.059 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:16 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:54:17.060 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:16 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:54:18.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:18 smithi123 ceph-mon[24496]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-03T08:54:19.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:18 smithi032 ceph-mon[22403]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-03T08:54:19.773 INFO:teuthology.orchestra.run.smithi123.stdout:Created osd(s) 6 on host 'smithi123' 2024-05-03T08:54:20.735 DEBUG:teuthology.orchestra.run.smithi123:osd.6> sudo journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.6.service 2024-05-03T08:54:20.737 INFO:tasks.cephadm:Deploying osd.7 on smithi123 with /dev/vg_nvme/lv_1... 2024-05-03T08:54:20.738 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-05-03T08:54:20.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:20 smithi123 ceph-mon[24496]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-03T08:54:20.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:20 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:20.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:20 smithi123 ceph-mon[24496]: from='osd.6 [v2:172.21.15.123:6816/2755860699,v1:172.21.15.123:6817/2755860699]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-03T08:54:20.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:20 smithi123 ceph-mon[24496]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-03T08:54:21.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:20 smithi032 ceph-mon[22403]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-03T08:54:21.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:20 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:21.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:20 smithi032 ceph-mon[22403]: from='osd.6 [v2:172.21.15.123:6816/2755860699,v1:172.21.15.123:6817/2755860699]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-03T08:54:21.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:20 smithi032 ceph-mon[22403]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-03T08:54:22.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:21 smithi032 ceph-mon[22403]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-03T08:54:22.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:21 smithi032 ceph-mon[22403]: osdmap e45: 7 total, 6 up, 7 in 2024-05-03T08:54:22.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:21 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-03T08:54:22.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:21 smithi032 ceph-mon[22403]: from='osd.6 [v2:172.21.15.123:6816/2755860699,v1:172.21.15.123:6817/2755860699]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-05-03T08:54:22.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:21 smithi032 ceph-mon[22403]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-05-03T08:54:22.211 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:21 smithi123 ceph-mon[24496]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-03T08:54:22.211 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:21 smithi123 ceph-mon[24496]: osdmap e45: 7 total, 6 up, 7 in 2024-05-03T08:54:22.211 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:21 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-03T08:54:22.211 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:21 smithi123 ceph-mon[24496]: from='osd.6 [v2:172.21.15.123:6816/2755860699,v1:172.21.15.123:6817/2755860699]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-05-03T08:54:22.212 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:21 smithi123 ceph-mon[24496]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-05-03T08:54:22.212 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:54:21 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:54:21.804+0000 7f7f2b9a9700 -1 osd.6 0 waiting for initial osdmap 2024-05-03T08:54:22.212 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:54:21 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:54:21.810+0000 7f7f2433c700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-03T08:54:22.958 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-05-03T08:54:22.958 INFO:teuthology.orchestra.run.smithi123.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-05-03T08:54:22.958 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 10+0 records in 2024-05-03T08:54:22.958 INFO:teuthology.orchestra.run.smithi123.stderr:10+0 records out 2024-05-03T08:54:22.958 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0135082 s, 776 MB/s 2024-05-03T08:54:22.958 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping successful for: 2024-05-03T08:54:23.031 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:22 smithi123 ceph-mon[24496]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-03T08:54:23.031 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:22 smithi123 ceph-mon[24496]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-05-03T08:54:23.031 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:22 smithi123 ceph-mon[24496]: osdmap e46: 7 total, 6 up, 7 in 2024-05-03T08:54:23.031 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:22 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-03T08:54:23.031 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:22 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-03T08:54:23.032 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:22 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:23.032 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:22 smithi123 ceph-mon[24496]: osd.6 [v2:172.21.15.123:6816/2755860699,v1:172.21.15.123:6817/2755860699] boot 2024-05-03T08:54:23.032 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:22 smithi123 ceph-mon[24496]: osdmap e47: 7 total, 7 up, 7 in 2024-05-03T08:54:23.032 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:22 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-03T08:54:23.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:22 smithi032 ceph-mon[22403]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-03T08:54:23.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:22 smithi032 ceph-mon[22403]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-05-03T08:54:23.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:22 smithi032 ceph-mon[22403]: osdmap e46: 7 total, 6 up, 7 in 2024-05-03T08:54:23.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:22 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-03T08:54:23.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:22 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-03T08:54:23.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:22 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:23.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:22 smithi032 ceph-mon[22403]: osd.6 [v2:172.21.15.123:6816/2755860699,v1:172.21.15.123:6817/2755860699] boot 2024-05-03T08:54:23.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:22 smithi032 ceph-mon[22403]: osdmap e47: 7 total, 7 up, 7 in 2024-05-03T08:54:23.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:22 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-03T08:54:23.608 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi123:vg_nvme/lv_1 2024-05-03T08:54:23.852 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:23 smithi123 ceph-mon[24496]: purged_snaps scrub starts 2024-05-03T08:54:23.853 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:23 smithi123 ceph-mon[24496]: purged_snaps scrub ok 2024-05-03T08:54:23.853 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:23 smithi123 ceph-mon[24496]: osdmap e48: 7 total, 7 up, 7 in 2024-05-03T08:54:24.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:23 smithi032 ceph-mon[22403]: purged_snaps scrub starts 2024-05-03T08:54:24.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:23 smithi032 ceph-mon[22403]: purged_snaps scrub ok 2024-05-03T08:54:24.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:23 smithi032 ceph-mon[22403]: osdmap e48: 7 total, 7 up, 7 in 2024-05-03T08:54:25.078 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:24 smithi123 ceph-mon[24496]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-03T08:54:25.078 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:24 smithi123 ceph-mon[24496]: osdmap e49: 7 total, 7 up, 7 in 2024-05-03T08:54:25.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:24 smithi032 ceph-mon[22403]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-03T08:54:25.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:24 smithi032 ceph-mon[22403]: osdmap e49: 7 total, 7 up, 7 in 2024-05-03T08:54:27.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:26 smithi032 ceph-mon[22403]: pgmap v149: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-03T08:54:27.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:26 smithi032 ceph-mon[22403]: osdmap e50: 7 total, 7 up, 7 in 2024-05-03T08:54:27.164 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:26 smithi123 ceph-mon[24496]: pgmap v149: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-03T08:54:27.164 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:26 smithi123 ceph-mon[24496]: osdmap e50: 7 total, 7 up, 7 in 2024-05-03T08:54:28.034 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:27 smithi123 ceph-mon[24496]: osdmap e51: 7 total, 7 up, 7 in 2024-05-03T08:54:28.034 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:27 smithi123 ceph-mon[24496]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:54:28.034 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:27 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-03T08:54:28.035 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:27 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-03T08:54:28.035 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:27 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:54:28.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:27 smithi032 ceph-mon[22403]: osdmap e51: 7 total, 7 up, 7 in 2024-05-03T08:54:28.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:27 smithi032 ceph-mon[22403]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:54:28.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:27 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-03T08:54:28.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:27 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-03T08:54:28.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:27 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:54:29.067 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:28 smithi123 ceph-mon[24496]: pgmap v152: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-03T08:54:29.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:28 smithi032 ceph-mon[22403]: pgmap v152: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-03T08:54:30.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:29 smithi032 ceph-mon[22403]: Detected new or changed devices on smithi123 2024-05-03T08:54:30.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:29 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:30.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:29 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:54:30.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:29 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:54:30.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:29 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:54:30.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:29 smithi032 ceph-mon[22403]: Adjusting osd_memory_target on smithi123 to 5312M 2024-05-03T08:54:30.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:29 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:30.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:29 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:30.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:29 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:54:30.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:29 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:54:30.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:29 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:54:30.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:29 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:30.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:29 smithi032 ceph-mon[22403]: from='client.? 172.21.15.123:0/1839310415' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b4464059-8168-4ed4-bb96-eb2b2dc4b248"}]: dispatch 2024-05-03T08:54:30.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:29 smithi032 ceph-mon[22403]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b4464059-8168-4ed4-bb96-eb2b2dc4b248"}]: dispatch 2024-05-03T08:54:30.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:29 smithi032 ceph-mon[22403]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b4464059-8168-4ed4-bb96-eb2b2dc4b248"}]': finished 2024-05-03T08:54:30.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:29 smithi032 ceph-mon[22403]: osdmap e52: 8 total, 7 up, 8 in 2024-05-03T08:54:30.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:29 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-03T08:54:30.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:29 smithi123 ceph-mon[24496]: Detected new or changed devices on smithi123 2024-05-03T08:54:30.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:29 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:30.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:29 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:54:30.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:29 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:54:30.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:29 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:54:30.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:29 smithi123 ceph-mon[24496]: Adjusting osd_memory_target on smithi123 to 5312M 2024-05-03T08:54:30.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:29 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:30.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:29 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:30.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:29 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:54:30.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:29 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:54:30.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:29 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:54:30.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:29 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:30.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:29 smithi123 ceph-mon[24496]: from='client.? 172.21.15.123:0/1839310415' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b4464059-8168-4ed4-bb96-eb2b2dc4b248"}]: dispatch 2024-05-03T08:54:30.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:29 smithi123 ceph-mon[24496]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b4464059-8168-4ed4-bb96-eb2b2dc4b248"}]: dispatch 2024-05-03T08:54:30.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:29 smithi123 ceph-mon[24496]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b4464059-8168-4ed4-bb96-eb2b2dc4b248"}]': finished 2024-05-03T08:54:30.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:29 smithi123 ceph-mon[24496]: osdmap e52: 8 total, 7 up, 8 in 2024-05-03T08:54:30.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:29 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-03T08:54:31.129 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:30 smithi123 ceph-mon[24496]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-05-03T08:54:31.129 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:30 smithi123 ceph-mon[24496]: from='client.? 172.21.15.123:0/181742251' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-03T08:54:31.137 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:30 smithi032 ceph-mon[22403]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-05-03T08:54:31.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:30 smithi032 ceph-mon[22403]: from='client.? 172.21.15.123:0/181742251' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-03T08:54:33.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:32 smithi032 ceph-mon[22403]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-03T08:54:33.223 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:32 smithi123 ceph-mon[24496]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-03T08:54:35.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:34 smithi032 ceph-mon[22403]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-03T08:54:35.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:34 smithi123 ceph-mon[24496]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-03T08:54:37.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:36 smithi032 ceph-mon[22403]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-03T08:54:37.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:36 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-03T08:54:37.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:36 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:54:37.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:36 smithi032 ceph-mon[22403]: Deploying daemon osd.7 on smithi123 2024-05-03T08:54:37.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:36 smithi123 ceph-mon[24496]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-03T08:54:37.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:36 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-03T08:54:37.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:36 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:54:37.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:36 smithi123 ceph-mon[24496]: Deploying daemon osd.7 on smithi123 2024-05-03T08:54:38.885 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:38 smithi123 ceph-mon[24496]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-03T08:54:39.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:38 smithi032 ceph-mon[22403]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-03T08:54:41.039 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:40 smithi123 ceph-mon[24496]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-03T08:54:41.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:40 smithi032 ceph-mon[22403]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-03T08:54:42.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:42.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:54:42.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:54:42.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:54:42.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:54:42.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:54:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:54:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:54:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:54:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:54:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:54:43.423 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:43 smithi123 ceph-mon[24496]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-03T08:54:43.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:43 smithi032 ceph-mon[22403]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-03T08:54:44.267 INFO:teuthology.orchestra.run.smithi123.stdout:Created osd(s) 7 on host 'smithi123' 2024-05-03T08:54:44.905 DEBUG:teuthology.orchestra.run.smithi123:osd.7> sudo journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.7.service 2024-05-03T08:54:44.907 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-05-03T08:54:44.907 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph osd stat -f json 2024-05-03T08:54:45.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:45 smithi123 ceph-mon[24496]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-03T08:54:45.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:45 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:45.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:45 smithi123 ceph-mon[24496]: from='osd.7 [v2:172.21.15.123:6824/2145148492,v1:172.21.15.123:6825/2145148492]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-03T08:54:45.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:45 smithi123 ceph-mon[24496]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-03T08:54:45.513 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:45 smithi032 ceph-mon[22403]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-03T08:54:45.513 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:45 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:45.513 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:45 smithi032 ceph-mon[22403]: from='osd.7 [v2:172.21.15.123:6824/2145148492,v1:172.21.15.123:6825/2145148492]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-03T08:54:45.513 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:45 smithi032 ceph-mon[22403]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-03T08:54:46.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:46 smithi123 ceph-mon[24496]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-03T08:54:46.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:46 smithi123 ceph-mon[24496]: osdmap e53: 8 total, 7 up, 8 in 2024-05-03T08:54:46.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:46 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-03T08:54:46.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:46 smithi123 ceph-mon[24496]: from='osd.7 [v2:172.21.15.123:6824/2145148492,v1:172.21.15.123:6825/2145148492]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-05-03T08:54:46.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:46 smithi123 ceph-mon[24496]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-05-03T08:54:46.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:46 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:46.478 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:54:46 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:54:46.277+0000 7fc7e1415700 -1 osd.7 0 waiting for initial osdmap 2024-05-03T08:54:46.478 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:54:46 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:54:46.284+0000 7fc7dd5af700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-03T08:54:46.488 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:46 smithi032 ceph-mon[22403]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-03T08:54:46.489 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:46 smithi032 ceph-mon[22403]: osdmap e53: 8 total, 7 up, 8 in 2024-05-03T08:54:46.489 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:46 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-03T08:54:46.489 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:46 smithi032 ceph-mon[22403]: from='osd.7 [v2:172.21.15.123:6824/2145148492,v1:172.21.15.123:6825/2145148492]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-05-03T08:54:46.489 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:46 smithi032 ceph-mon[22403]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-05-03T08:54:46.489 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:46 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:46.930 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:54:47.438 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:47 smithi123 ceph-mon[24496]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-03T08:54:47.438 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:47 smithi123 ceph-mon[24496]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-05-03T08:54:47.438 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:47 smithi123 ceph-mon[24496]: osdmap e54: 8 total, 7 up, 8 in 2024-05-03T08:54:47.438 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:47 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-03T08:54:47.438 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:47 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-03T08:54:47.438 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:47 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/2748729247' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-03T08:54:47.574 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:47 smithi032 ceph-mon[22403]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-03T08:54:47.574 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:47 smithi032 ceph-mon[22403]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-05-03T08:54:47.574 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:47 smithi032 ceph-mon[22403]: osdmap e54: 8 total, 7 up, 8 in 2024-05-03T08:54:47.574 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:47 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-03T08:54:47.574 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:47 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-03T08:54:47.574 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:47 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/2748729247' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-03T08:54:47.603 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1714726462,"num_in_osds":8,"osd_in_since":1714726469,"num_remapped_pgs":0} 2024-05-03T08:54:48.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:48 smithi123 ceph-mon[24496]: purged_snaps scrub starts 2024-05-03T08:54:48.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:48 smithi123 ceph-mon[24496]: purged_snaps scrub ok 2024-05-03T08:54:48.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:48 smithi123 ceph-mon[24496]: osd.7 [v2:172.21.15.123:6824/2145148492,v1:172.21.15.123:6825/2145148492] boot 2024-05-03T08:54:48.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:48 smithi123 ceph-mon[24496]: osdmap e55: 8 total, 8 up, 8 in 2024-05-03T08:54:48.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:48 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-03T08:54:48.604 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph osd stat -f json 2024-05-03T08:54:48.629 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:48 smithi032 ceph-mon[22403]: purged_snaps scrub starts 2024-05-03T08:54:48.629 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:48 smithi032 ceph-mon[22403]: purged_snaps scrub ok 2024-05-03T08:54:48.629 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:48 smithi032 ceph-mon[22403]: osd.7 [v2:172.21.15.123:6824/2145148492,v1:172.21.15.123:6825/2145148492] boot 2024-05-03T08:54:48.629 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:48 smithi032 ceph-mon[22403]: osdmap e55: 8 total, 8 up, 8 in 2024-05-03T08:54:48.629 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:48 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-03T08:54:49.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:49 smithi032 ceph-mon[22403]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:54:49.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:49 smithi032 ceph-mon[22403]: osdmap e56: 8 total, 8 up, 8 in 2024-05-03T08:54:49.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:49.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:54:49.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:54:49.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:54:49.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:54:49.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:49.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:49.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:54:49.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:54:49.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:54:49.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:49 smithi123 ceph-mon[24496]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:54:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:49 smithi123 ceph-mon[24496]: osdmap e56: 8 total, 8 up, 8 in 2024-05-03T08:54:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:54:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:54:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:54:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-03T08:54:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:54:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:54:49.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:54:49.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:54:50.557 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:54:50.558 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:50 smithi032 ceph-mon[22403]: Detected new or changed devices on smithi123 2024-05-03T08:54:50.558 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:50 smithi032 ceph-mon[22403]: Adjusting osd_memory_target on smithi123 to 3984M 2024-05-03T08:54:50.558 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:50 smithi032 ceph-mon[22403]: osdmap e57: 8 total, 8 up, 8 in 2024-05-03T08:54:50.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:50 smithi123 ceph-mon[24496]: Detected new or changed devices on smithi123 2024-05-03T08:54:50.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:50 smithi123 ceph-mon[24496]: Adjusting osd_memory_target on smithi123 to 3984M 2024-05-03T08:54:50.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:50 smithi123 ceph-mon[24496]: osdmap e57: 8 total, 8 up, 8 in 2024-05-03T08:54:51.260 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":58,"num_osds":8,"num_up_osds":8,"osd_up_since":1714726487,"num_in_osds":8,"osd_in_since":1714726469,"num_remapped_pgs":0} 2024-05-03T08:54:51.260 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph osd dump --format=json 2024-05-03T08:54:51.531 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:54:51.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:51 smithi032 ceph-mon[22403]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:54:51.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:51 smithi032 ceph-mon[22403]: osdmap e58: 8 total, 8 up, 8 in 2024-05-03T08:54:51.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:51 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/1429077816' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-03T08:54:51.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:51 smithi123 ceph-mon[24496]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:54:51.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:51 smithi123 ceph-mon[24496]: osdmap e58: 8 total, 8 up, 8 in 2024-05-03T08:54:51.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:51 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/1429077816' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-03T08:54:52.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:52 smithi032 ceph-mon[22403]: osdmap e59: 8 total, 8 up, 8 in 2024-05-03T08:54:52.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:52 smithi123 ceph-mon[24496]: osdmap e59: 8 total, 8 up, 8 in 2024-05-03T08:54:53.426 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:54:53.426 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":59,"fsid":"df63fec2-0929-11ef-bc97-c7b262605968","created":"2024-05-03T08:48:46.312309+0000","modified":"2024-05-03T08:54:51.296187+0000","last_up_change":"2024-05-03T08:54:47.272166+0000","last_in_change":"2024-05-03T08:54:29.806611+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-03T08:52:48.233719+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"39f89f27-f5a3-4b54-8b6e-cb5e8ef45738","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":2936167065},{"type":"v1","addr":"172.21.15.32:6803","nonce":2936167065}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":2936167065},{"type":"v1","addr":"172.21.15.32:6805","nonce":2936167065}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":2936167065},{"type":"v1","addr":"172.21.15.32:6809","nonce":2936167065}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":2936167065},{"type":"v1","addr":"172.21.15.32:6807","nonce":2936167065}]},"public_addr":"172.21.15.32:6803/2936167065","cluster_addr":"172.21.15.32:6805/2936167065","heartbeat_back_addr":"172.21.15.32:6809/2936167065","heartbeat_front_addr":"172.21.15.32:6807/2936167065","state":["exists","up"]},{"osd":1,"uuid":"2bc68418-2844-49df-8c1f-016dbbe3d34d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":4253870677},{"type":"v1","addr":"172.21.15.32:6811","nonce":4253870677}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":4253870677},{"type":"v1","addr":"172.21.15.32:6813","nonce":4253870677}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":4253870677},{"type":"v1","addr":"172.21.15.32:6817","nonce":4253870677}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":4253870677},{"type":"v1","addr":"172.21.15.32:6815","nonce":4253870677}]},"public_addr":"172.21.15.32:6811/4253870677","cluster_addr":"172.21.15.32:6813/4253870677","heartbeat_back_addr":"172.21.15.32:6817/4253870677","heartbeat_front_addr":"172.21.15.32:6815/4253870677","state":["exists","up"]},{"osd":2,"uuid":"9395950f-d4fa-4dd7-8c2b-61e5f27aabef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3583854825},{"type":"v1","addr":"172.21.15.32:6819","nonce":3583854825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3583854825},{"type":"v1","addr":"172.21.15.32:6821","nonce":3583854825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3583854825},{"type":"v1","addr":"172.21.15.32:6825","nonce":3583854825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3583854825},{"type":"v1","addr":"172.21.15.32:6823","nonce":3583854825}]},"public_addr":"172.21.15.32:6819/3583854825","cluster_addr":"172.21.15.32:6821/3583854825","heartbeat_back_addr":"172.21.15.32:6825/3583854825","heartbeat_front_addr":"172.21.15.32:6823/3583854825","state":["exists","up"]},{"osd":3,"uuid":"e68bbfc1-e00a-46fd-a30d-8047f56633e3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":3336131837},{"type":"v1","addr":"172.21.15.32:6827","nonce":3336131837}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":3336131837},{"type":"v1","addr":"172.21.15.32:6829","nonce":3336131837}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":3336131837},{"type":"v1","addr":"172.21.15.32:6833","nonce":3336131837}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":3336131837},{"type":"v1","addr":"172.21.15.32:6831","nonce":3336131837}]},"public_addr":"172.21.15.32:6827/3336131837","cluster_addr":"172.21.15.32:6829/3336131837","heartbeat_back_addr":"172.21.15.32:6833/3336131837","heartbeat_front_addr":"172.21.15.32:6831/3336131837","state":["exists","up"]},{"osd":4,"uuid":"84c5c1ee-8601-4517-8ac8-7312fa8a899e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6800","nonce":309107390},{"type":"v1","addr":"172.21.15.123:6801","nonce":309107390}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6802","nonce":309107390},{"type":"v1","addr":"172.21.15.123:6803","nonce":309107390}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6806","nonce":309107390},{"type":"v1","addr":"172.21.15.123:6807","nonce":309107390}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6804","nonce":309107390},{"type":"v1","addr":"172.21.15.123:6805","nonce":309107390}]},"public_addr":"172.21.15.123:6801/309107390","cluster_addr":"172.21.15.123:6803/309107390","heartbeat_back_addr":"172.21.15.123:6807/309107390","heartbeat_front_addr":"172.21.15.123:6805/309107390","state":["exists","up"]},{"osd":5,"uuid":"6cdcf768-83e1-48a6-88d5-2b01d7efb20c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6808","nonce":1030990683},{"type":"v1","addr":"172.21.15.123:6809","nonce":1030990683}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6810","nonce":1030990683},{"type":"v1","addr":"172.21.15.123:6811","nonce":1030990683}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6814","nonce":1030990683},{"type":"v1","addr":"172.21.15.123:6815","nonce":1030990683}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6812","nonce":1030990683},{"type":"v1","addr":"172.21.15.123:6813","nonce":1030990683}]},"public_addr":"172.21.15.123:6809/1030990683","cluster_addr":"172.21.15.123:6811/1030990683","heartbeat_back_addr":"172.21.15.123:6815/1030990683","heartbeat_front_addr":"172.21.15.123:6813/1030990683","state":["exists","up"]},{"osd":6,"uuid":"526d3fa7-1ba8-4921-8caa-86b6f9643543","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6816","nonce":2755860699},{"type":"v1","addr":"172.21.15.123:6817","nonce":2755860699}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6818","nonce":2755860699},{"type":"v1","addr":"172.21.15.123:6819","nonce":2755860699}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6822","nonce":2755860699},{"type":"v1","addr":"172.21.15.123:6823","nonce":2755860699}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6820","nonce":2755860699},{"type":"v1","addr":"172.21.15.123:6821","nonce":2755860699}]},"public_addr":"172.21.15.123:6817/2755860699","cluster_addr":"172.21.15.123:6819/2755860699","heartbeat_back_addr":"172.21.15.123:6823/2755860699","heartbeat_front_addr":"172.21.15.123:6821/2755860699","state":["exists","up"]},{"osd":7,"uuid":"b4464059-8168-4ed4-bb96-eb2b2dc4b248","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6824","nonce":2145148492},{"type":"v1","addr":"172.21.15.123:6825","nonce":2145148492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6826","nonce":2145148492},{"type":"v1","addr":"172.21.15.123:6827","nonce":2145148492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6830","nonce":2145148492},{"type":"v1","addr":"172.21.15.123:6831","nonce":2145148492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6828","nonce":2145148492},{"type":"v1","addr":"172.21.15.123:6829","nonce":2145148492}]},"public_addr":"172.21.15.123:6825/2145148492","cluster_addr":"172.21.15.123:6827/2145148492","heartbeat_back_addr":"172.21.15.123:6831/2145148492","heartbeat_front_addr":"172.21.15.123:6829/2145148492","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:51:47.792408+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:52:18.852747+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:52:45.224445+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:53:11.827644+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:53:33.635569+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:53:56.691739+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:54:20.799963+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:54:45.292947+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/3998187481":"2024-05-04T08:50:40.741983+0000","172.21.15.32:6801/4241898604":"2024-05-04T08:50:40.741983+0000","172.21.15.32:0/244682197":"2024-05-04T08:49:49.012875+0000","172.21.15.32:0/3672752227":"2024-05-04T08:49:49.012875+0000","172.21.15.32:0/2700929476":"2024-05-04T08:50:40.741983+0000","172.21.15.32:6800/3931596567":"2024-05-04T08:49:49.012875+0000","172.21.15.32:0/1984734425":"2024-05-04T08:49:49.012875+0000","172.21.15.32:0/616547951":"2024-05-04T08:49:18.973825+0000","172.21.15.32:0/383308891":"2024-05-04T08:50:40.741983+0000","172.21.15.32:6801/3931596567":"2024-05-04T08:49:49.012875+0000","172.21.15.32:6800/1218624955":"2024-05-04T08:49:18.973825+0000","172.21.15.32:6801/1218624955":"2024-05-04T08:49:18.973825+0000","172.21.15.32:6800/4241898604":"2024-05-04T08:50:40.741983+0000","172.21.15.32:0/919874771":"2024-05-04T08:49:18.973825+0000","172.21.15.32:0/2086397669":"2024-05-04T08:49:18.973825+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-03T08:54:53.637 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:53 smithi032 ceph-mon[22403]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:54:53.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:53 smithi123 ceph-mon[24496]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:54:54.291 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-05-03T08:52:48.233719+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-05-03T08:54:54.292 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-05-03T08:54:54.523 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:54:54.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:54 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/2233884795' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-03T08:54:54.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:54 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/2233884795' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-03T08:54:55.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:55 smithi032 ceph-mon[22403]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-05-03T08:54:55.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:55 smithi123 ceph-mon[24496]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-05-03T08:54:56.471 INFO:teuthology.orchestra.run.smithi032.stdout:pg_num: 1 2024-05-03T08:54:57.186 INFO:tasks.cephadm:Setting up client nodes... 2024-05-03T08:54:57.186 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-05-03T08:54:57.501 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:57 smithi032 ceph-mon[22403]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-05-03T08:54:57.501 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:57 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/1732348297' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-03T08:54:57.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:57 smithi123 ceph-mon[24496]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-05-03T08:54:57.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:57 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/1732348297' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-03T08:54:59.325 INFO:teuthology.orchestra.run.smithi032.stdout:[client.0] 2024-05-03T08:54:59.325 INFO:teuthology.orchestra.run.smithi032.stdout: key = AQBjpjRmPoQeExAAc9ctgqUs8QNHkissRV7j3w== 2024-05-03T08:54:59.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:59 smithi032 ceph-mon[22403]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-03T08:54:59.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:54:59 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/1274980728' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-03T08:54:59.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:59 smithi123 ceph-mon[24496]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-03T08:54:59.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:54:59 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/1274980728' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-03T08:55:00.146 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-05-03T08:55:00.146 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-05-03T08:55:00.146 DEBUG:teuthology.orchestra.run.smithi032:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-05-03T08:55:00.186 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-05-03T08:55:00.187 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-05-03T08:55:00.187 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph mgr dump --format=json 2024-05-03T08:55:00.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:00 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/1274980728' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-03T08:55:00.451 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:00.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:00 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/1274980728' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-03T08:55:01.605 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:01 smithi032 ceph-mon[22403]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-05-03T08:55:01.628 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:01 smithi123 ceph-mon[24496]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-05-03T08:55:02.365 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:55:03.177 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":19,"active_gid":14202,"active_name":"smithi032.umllay","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6800","nonce":422375101},{"type":"v1","addr":"172.21.15.32:6801","nonce":422375101}]},"active_addr":"172.21.15.32:6801/422375101","active_change":"2024-05-03T08:50:40.742327+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14220,"name":"smithi123.xrxinh","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.32:8443/","prometheus":"http://172.21.15.32:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":2162696854}]},{"addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":4910454}]},{"addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":1233564317}]},{"addrvec":[{"type":"v2","addr":"172.21.15.32:0","nonce":1480225686}]}]}} 2024-05-03T08:55:03.184 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-05-03T08:55:03.184 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-05-03T08:55:03.185 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph osd dump --format=json 2024-05-03T08:55:03.412 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:03.535 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:03 smithi032 ceph-mon[22403]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-05-03T08:55:03.535 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:03 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/3635426612' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-03T08:55:03.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:03 smithi123 ceph-mon[24496]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-05-03T08:55:03.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:03 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/3635426612' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-03T08:55:05.368 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:55:05.368 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":59,"fsid":"df63fec2-0929-11ef-bc97-c7b262605968","created":"2024-05-03T08:48:46.312309+0000","modified":"2024-05-03T08:54:51.296187+0000","last_up_change":"2024-05-03T08:54:47.272166+0000","last_in_change":"2024-05-03T08:54:29.806611+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-03T08:52:48.233719+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"39f89f27-f5a3-4b54-8b6e-cb5e8ef45738","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":2936167065},{"type":"v1","addr":"172.21.15.32:6803","nonce":2936167065}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":2936167065},{"type":"v1","addr":"172.21.15.32:6805","nonce":2936167065}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":2936167065},{"type":"v1","addr":"172.21.15.32:6809","nonce":2936167065}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":2936167065},{"type":"v1","addr":"172.21.15.32:6807","nonce":2936167065}]},"public_addr":"172.21.15.32:6803/2936167065","cluster_addr":"172.21.15.32:6805/2936167065","heartbeat_back_addr":"172.21.15.32:6809/2936167065","heartbeat_front_addr":"172.21.15.32:6807/2936167065","state":["exists","up"]},{"osd":1,"uuid":"2bc68418-2844-49df-8c1f-016dbbe3d34d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":4253870677},{"type":"v1","addr":"172.21.15.32:6811","nonce":4253870677}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":4253870677},{"type":"v1","addr":"172.21.15.32:6813","nonce":4253870677}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":4253870677},{"type":"v1","addr":"172.21.15.32:6817","nonce":4253870677}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":4253870677},{"type":"v1","addr":"172.21.15.32:6815","nonce":4253870677}]},"public_addr":"172.21.15.32:6811/4253870677","cluster_addr":"172.21.15.32:6813/4253870677","heartbeat_back_addr":"172.21.15.32:6817/4253870677","heartbeat_front_addr":"172.21.15.32:6815/4253870677","state":["exists","up"]},{"osd":2,"uuid":"9395950f-d4fa-4dd7-8c2b-61e5f27aabef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3583854825},{"type":"v1","addr":"172.21.15.32:6819","nonce":3583854825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3583854825},{"type":"v1","addr":"172.21.15.32:6821","nonce":3583854825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3583854825},{"type":"v1","addr":"172.21.15.32:6825","nonce":3583854825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3583854825},{"type":"v1","addr":"172.21.15.32:6823","nonce":3583854825}]},"public_addr":"172.21.15.32:6819/3583854825","cluster_addr":"172.21.15.32:6821/3583854825","heartbeat_back_addr":"172.21.15.32:6825/3583854825","heartbeat_front_addr":"172.21.15.32:6823/3583854825","state":["exists","up"]},{"osd":3,"uuid":"e68bbfc1-e00a-46fd-a30d-8047f56633e3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":3336131837},{"type":"v1","addr":"172.21.15.32:6827","nonce":3336131837}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":3336131837},{"type":"v1","addr":"172.21.15.32:6829","nonce":3336131837}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":3336131837},{"type":"v1","addr":"172.21.15.32:6833","nonce":3336131837}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":3336131837},{"type":"v1","addr":"172.21.15.32:6831","nonce":3336131837}]},"public_addr":"172.21.15.32:6827/3336131837","cluster_addr":"172.21.15.32:6829/3336131837","heartbeat_back_addr":"172.21.15.32:6833/3336131837","heartbeat_front_addr":"172.21.15.32:6831/3336131837","state":["exists","up"]},{"osd":4,"uuid":"84c5c1ee-8601-4517-8ac8-7312fa8a899e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6800","nonce":309107390},{"type":"v1","addr":"172.21.15.123:6801","nonce":309107390}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6802","nonce":309107390},{"type":"v1","addr":"172.21.15.123:6803","nonce":309107390}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6806","nonce":309107390},{"type":"v1","addr":"172.21.15.123:6807","nonce":309107390}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6804","nonce":309107390},{"type":"v1","addr":"172.21.15.123:6805","nonce":309107390}]},"public_addr":"172.21.15.123:6801/309107390","cluster_addr":"172.21.15.123:6803/309107390","heartbeat_back_addr":"172.21.15.123:6807/309107390","heartbeat_front_addr":"172.21.15.123:6805/309107390","state":["exists","up"]},{"osd":5,"uuid":"6cdcf768-83e1-48a6-88d5-2b01d7efb20c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6808","nonce":1030990683},{"type":"v1","addr":"172.21.15.123:6809","nonce":1030990683}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6810","nonce":1030990683},{"type":"v1","addr":"172.21.15.123:6811","nonce":1030990683}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6814","nonce":1030990683},{"type":"v1","addr":"172.21.15.123:6815","nonce":1030990683}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6812","nonce":1030990683},{"type":"v1","addr":"172.21.15.123:6813","nonce":1030990683}]},"public_addr":"172.21.15.123:6809/1030990683","cluster_addr":"172.21.15.123:6811/1030990683","heartbeat_back_addr":"172.21.15.123:6815/1030990683","heartbeat_front_addr":"172.21.15.123:6813/1030990683","state":["exists","up"]},{"osd":6,"uuid":"526d3fa7-1ba8-4921-8caa-86b6f9643543","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6816","nonce":2755860699},{"type":"v1","addr":"172.21.15.123:6817","nonce":2755860699}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6818","nonce":2755860699},{"type":"v1","addr":"172.21.15.123:6819","nonce":2755860699}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6822","nonce":2755860699},{"type":"v1","addr":"172.21.15.123:6823","nonce":2755860699}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6820","nonce":2755860699},{"type":"v1","addr":"172.21.15.123:6821","nonce":2755860699}]},"public_addr":"172.21.15.123:6817/2755860699","cluster_addr":"172.21.15.123:6819/2755860699","heartbeat_back_addr":"172.21.15.123:6823/2755860699","heartbeat_front_addr":"172.21.15.123:6821/2755860699","state":["exists","up"]},{"osd":7,"uuid":"b4464059-8168-4ed4-bb96-eb2b2dc4b248","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6824","nonce":2145148492},{"type":"v1","addr":"172.21.15.123:6825","nonce":2145148492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6826","nonce":2145148492},{"type":"v1","addr":"172.21.15.123:6827","nonce":2145148492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6830","nonce":2145148492},{"type":"v1","addr":"172.21.15.123:6831","nonce":2145148492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6828","nonce":2145148492},{"type":"v1","addr":"172.21.15.123:6829","nonce":2145148492}]},"public_addr":"172.21.15.123:6825/2145148492","cluster_addr":"172.21.15.123:6827/2145148492","heartbeat_back_addr":"172.21.15.123:6831/2145148492","heartbeat_front_addr":"172.21.15.123:6829/2145148492","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:51:47.792408+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:52:18.852747+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:52:45.224445+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:53:11.827644+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:53:33.635569+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:53:56.691739+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:54:20.799963+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:54:45.292947+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/3998187481":"2024-05-04T08:50:40.741983+0000","172.21.15.32:6801/4241898604":"2024-05-04T08:50:40.741983+0000","172.21.15.32:0/244682197":"2024-05-04T08:49:49.012875+0000","172.21.15.32:0/3672752227":"2024-05-04T08:49:49.012875+0000","172.21.15.32:0/2700929476":"2024-05-04T08:50:40.741983+0000","172.21.15.32:6800/3931596567":"2024-05-04T08:49:49.012875+0000","172.21.15.32:0/1984734425":"2024-05-04T08:49:49.012875+0000","172.21.15.32:0/616547951":"2024-05-04T08:49:18.973825+0000","172.21.15.32:0/383308891":"2024-05-04T08:50:40.741983+0000","172.21.15.32:6801/3931596567":"2024-05-04T08:49:49.012875+0000","172.21.15.32:6800/1218624955":"2024-05-04T08:49:18.973825+0000","172.21.15.32:6801/1218624955":"2024-05-04T08:49:18.973825+0000","172.21.15.32:6800/4241898604":"2024-05-04T08:50:40.741983+0000","172.21.15.32:0/919874771":"2024-05-04T08:49:18.973825+0000","172.21.15.32:0/2086397669":"2024-05-04T08:49:18.973825+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-03T08:55:05.382 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:05 smithi032 ceph-mon[22403]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-05-03T08:55:05.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:05 smithi123 ceph-mon[24496]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-05-03T08:55:06.147 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-05-03T08:55:06.147 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph osd dump --format=json 2024-05-03T08:55:06.371 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:06.614 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:06 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/1687980249' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-03T08:55:06.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:06 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/1687980249' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-03T08:55:07.529 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:07 smithi032 ceph-mon[22403]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:07.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:07 smithi123 ceph-mon[24496]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:08.214 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:55:08.214 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":59,"fsid":"df63fec2-0929-11ef-bc97-c7b262605968","created":"2024-05-03T08:48:46.312309+0000","modified":"2024-05-03T08:54:51.296187+0000","last_up_change":"2024-05-03T08:54:47.272166+0000","last_in_change":"2024-05-03T08:54:29.806611+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-03T08:52:48.233719+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"39f89f27-f5a3-4b54-8b6e-cb5e8ef45738","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":2936167065},{"type":"v1","addr":"172.21.15.32:6803","nonce":2936167065}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":2936167065},{"type":"v1","addr":"172.21.15.32:6805","nonce":2936167065}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6808","nonce":2936167065},{"type":"v1","addr":"172.21.15.32:6809","nonce":2936167065}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":2936167065},{"type":"v1","addr":"172.21.15.32:6807","nonce":2936167065}]},"public_addr":"172.21.15.32:6803/2936167065","cluster_addr":"172.21.15.32:6805/2936167065","heartbeat_back_addr":"172.21.15.32:6809/2936167065","heartbeat_front_addr":"172.21.15.32:6807/2936167065","state":["exists","up"]},{"osd":1,"uuid":"2bc68418-2844-49df-8c1f-016dbbe3d34d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6810","nonce":4253870677},{"type":"v1","addr":"172.21.15.32:6811","nonce":4253870677}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6812","nonce":4253870677},{"type":"v1","addr":"172.21.15.32:6813","nonce":4253870677}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6816","nonce":4253870677},{"type":"v1","addr":"172.21.15.32:6817","nonce":4253870677}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6814","nonce":4253870677},{"type":"v1","addr":"172.21.15.32:6815","nonce":4253870677}]},"public_addr":"172.21.15.32:6811/4253870677","cluster_addr":"172.21.15.32:6813/4253870677","heartbeat_back_addr":"172.21.15.32:6817/4253870677","heartbeat_front_addr":"172.21.15.32:6815/4253870677","state":["exists","up"]},{"osd":2,"uuid":"9395950f-d4fa-4dd7-8c2b-61e5f27aabef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6818","nonce":3583854825},{"type":"v1","addr":"172.21.15.32:6819","nonce":3583854825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6820","nonce":3583854825},{"type":"v1","addr":"172.21.15.32:6821","nonce":3583854825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6824","nonce":3583854825},{"type":"v1","addr":"172.21.15.32:6825","nonce":3583854825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6822","nonce":3583854825},{"type":"v1","addr":"172.21.15.32:6823","nonce":3583854825}]},"public_addr":"172.21.15.32:6819/3583854825","cluster_addr":"172.21.15.32:6821/3583854825","heartbeat_back_addr":"172.21.15.32:6825/3583854825","heartbeat_front_addr":"172.21.15.32:6823/3583854825","state":["exists","up"]},{"osd":3,"uuid":"e68bbfc1-e00a-46fd-a30d-8047f56633e3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6826","nonce":3336131837},{"type":"v1","addr":"172.21.15.32:6827","nonce":3336131837}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6828","nonce":3336131837},{"type":"v1","addr":"172.21.15.32:6829","nonce":3336131837}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6832","nonce":3336131837},{"type":"v1","addr":"172.21.15.32:6833","nonce":3336131837}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6830","nonce":3336131837},{"type":"v1","addr":"172.21.15.32:6831","nonce":3336131837}]},"public_addr":"172.21.15.32:6827/3336131837","cluster_addr":"172.21.15.32:6829/3336131837","heartbeat_back_addr":"172.21.15.32:6833/3336131837","heartbeat_front_addr":"172.21.15.32:6831/3336131837","state":["exists","up"]},{"osd":4,"uuid":"84c5c1ee-8601-4517-8ac8-7312fa8a899e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6800","nonce":309107390},{"type":"v1","addr":"172.21.15.123:6801","nonce":309107390}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6802","nonce":309107390},{"type":"v1","addr":"172.21.15.123:6803","nonce":309107390}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6806","nonce":309107390},{"type":"v1","addr":"172.21.15.123:6807","nonce":309107390}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6804","nonce":309107390},{"type":"v1","addr":"172.21.15.123:6805","nonce":309107390}]},"public_addr":"172.21.15.123:6801/309107390","cluster_addr":"172.21.15.123:6803/309107390","heartbeat_back_addr":"172.21.15.123:6807/309107390","heartbeat_front_addr":"172.21.15.123:6805/309107390","state":["exists","up"]},{"osd":5,"uuid":"6cdcf768-83e1-48a6-88d5-2b01d7efb20c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6808","nonce":1030990683},{"type":"v1","addr":"172.21.15.123:6809","nonce":1030990683}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6810","nonce":1030990683},{"type":"v1","addr":"172.21.15.123:6811","nonce":1030990683}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6814","nonce":1030990683},{"type":"v1","addr":"172.21.15.123:6815","nonce":1030990683}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6812","nonce":1030990683},{"type":"v1","addr":"172.21.15.123:6813","nonce":1030990683}]},"public_addr":"172.21.15.123:6809/1030990683","cluster_addr":"172.21.15.123:6811/1030990683","heartbeat_back_addr":"172.21.15.123:6815/1030990683","heartbeat_front_addr":"172.21.15.123:6813/1030990683","state":["exists","up"]},{"osd":6,"uuid":"526d3fa7-1ba8-4921-8caa-86b6f9643543","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6816","nonce":2755860699},{"type":"v1","addr":"172.21.15.123:6817","nonce":2755860699}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6818","nonce":2755860699},{"type":"v1","addr":"172.21.15.123:6819","nonce":2755860699}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6822","nonce":2755860699},{"type":"v1","addr":"172.21.15.123:6823","nonce":2755860699}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6820","nonce":2755860699},{"type":"v1","addr":"172.21.15.123:6821","nonce":2755860699}]},"public_addr":"172.21.15.123:6817/2755860699","cluster_addr":"172.21.15.123:6819/2755860699","heartbeat_back_addr":"172.21.15.123:6823/2755860699","heartbeat_front_addr":"172.21.15.123:6821/2755860699","state":["exists","up"]},{"osd":7,"uuid":"b4464059-8168-4ed4-bb96-eb2b2dc4b248","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6824","nonce":2145148492},{"type":"v1","addr":"172.21.15.123:6825","nonce":2145148492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6826","nonce":2145148492},{"type":"v1","addr":"172.21.15.123:6827","nonce":2145148492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6830","nonce":2145148492},{"type":"v1","addr":"172.21.15.123:6831","nonce":2145148492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6828","nonce":2145148492},{"type":"v1","addr":"172.21.15.123:6829","nonce":2145148492}]},"public_addr":"172.21.15.123:6825/2145148492","cluster_addr":"172.21.15.123:6827/2145148492","heartbeat_back_addr":"172.21.15.123:6831/2145148492","heartbeat_front_addr":"172.21.15.123:6829/2145148492","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:51:47.792408+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:52:18.852747+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:52:45.224445+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:53:11.827644+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:53:33.635569+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:53:56.691739+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:54:20.799963+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-03T08:54:45.292947+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.32:0/3998187481":"2024-05-04T08:50:40.741983+0000","172.21.15.32:6801/4241898604":"2024-05-04T08:50:40.741983+0000","172.21.15.32:0/244682197":"2024-05-04T08:49:49.012875+0000","172.21.15.32:0/3672752227":"2024-05-04T08:49:49.012875+0000","172.21.15.32:0/2700929476":"2024-05-04T08:50:40.741983+0000","172.21.15.32:6800/3931596567":"2024-05-04T08:49:49.012875+0000","172.21.15.32:0/1984734425":"2024-05-04T08:49:49.012875+0000","172.21.15.32:0/616547951":"2024-05-04T08:49:18.973825+0000","172.21.15.32:0/383308891":"2024-05-04T08:50:40.741983+0000","172.21.15.32:6801/3931596567":"2024-05-04T08:49:49.012875+0000","172.21.15.32:6800/1218624955":"2024-05-04T08:49:18.973825+0000","172.21.15.32:6801/1218624955":"2024-05-04T08:49:18.973825+0000","172.21.15.32:6800/4241898604":"2024-05-04T08:50:40.741983+0000","172.21.15.32:0/919874771":"2024-05-04T08:49:18.973825+0000","172.21.15.32:0/2086397669":"2024-05-04T08:49:18.973825+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-03T08:55:08.594 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:08 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/2415478470' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-03T08:55:08.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:08 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/2415478470' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-03T08:55:09.027 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-05-03T08:55:09.027 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-05-03T08:55:09.028 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-05-03T08:55:09.028 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-05-03T08:55:09.028 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-05-03T08:55:09.029 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-05-03T08:55:09.029 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-05-03T08:55:09.029 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-05-03T08:55:09.349 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:09 smithi032 ceph-mon[22403]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:09.437 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:09.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:09 smithi123 ceph-mon[24496]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:10.309 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:10.312 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:10.316 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:10.316 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:10.316 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:10.317 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:10.319 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:11.533 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:11 smithi032 ceph-mon[22403]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:11.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:11 smithi123 ceph-mon[24496]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:13.621 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:13 smithi032 ceph-mon[22403]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:13.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:13 smithi123 ceph-mon[24496]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:15.461 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:15 smithi032 ceph-mon[22403]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:15.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:15 smithi123 ceph-mon[24496]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:17.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:17 smithi032 ceph-mon[22403]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:17.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:17 smithi123 ceph-mon[24496]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:19.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:19 smithi032 ceph-mon[22403]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:19.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:19 smithi123 ceph-mon[24496]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:21.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:21 smithi032 ceph-mon[22403]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:21.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:21 smithi123 ceph-mon[24496]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:21.729 INFO:teuthology.orchestra.run.smithi032.stdout:133143986200 2024-05-03T08:55:21.729 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-05-03T08:55:22.057 INFO:teuthology.orchestra.run.smithi032.stdout:81604378657 2024-05-03T08:55:22.057 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-05-03T08:55:23.148 INFO:teuthology.orchestra.run.smithi032.stdout:38654705710 2024-05-03T08:55:23.148 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-05-03T08:55:23.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:23 smithi032 ceph-mon[22403]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:23.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:23 smithi123 ceph-mon[24496]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:24.168 INFO:teuthology.orchestra.run.smithi032.stdout:201863462926 2024-05-03T08:55:24.168 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-05-03T08:55:24.663 INFO:teuthology.orchestra.run.smithi032.stdout:111669149724 2024-05-03T08:55:24.664 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-05-03T08:55:25.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:25 smithi032 ceph-mon[22403]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:25.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:25 smithi123 ceph-mon[24496]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:26.106 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:26.106 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:26.555 INFO:teuthology.orchestra.run.smithi032.stdout:236223201290 2024-05-03T08:55:26.555 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-05-03T08:55:27.020 INFO:teuthology.orchestra.run.smithi032.stdout:60129542184 2024-05-03T08:55:27.020 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-05-03T08:55:27.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:27 smithi032 ceph-mon[22403]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:27.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:27 smithi123 ceph-mon[24496]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:28.167 INFO:teuthology.orchestra.run.smithi032.stdout:167503724565 2024-05-03T08:55:28.167 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-05-03T08:55:28.633 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:28.633 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:28.634 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:29.630 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:29 smithi032 ceph-mon[22403]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:29.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:29 smithi123 ceph-mon[24496]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:30.605 INFO:teuthology.orchestra.run.smithi032.stdout:81604378660 2024-05-03T08:55:31.577 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:31 smithi032 ceph-mon[22403]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:31.577 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:31 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/308000094' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-03T08:55:31.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:31 smithi123 ceph-mon[24496]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:31.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:31 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/308000094' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-03T08:55:31.907 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378657 got 81604378660 for osd.2 2024-05-03T08:55:31.907 DEBUG:teuthology.parallel:result is None 2024-05-03T08:55:33.336 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:33.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:33 smithi032 ceph-mon[22403]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:33.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:33 smithi123 ceph-mon[24496]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:34.241 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:34.242 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:34.441 INFO:teuthology.orchestra.run.smithi032.stdout:201863462929 2024-05-03T08:55:35.508 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:35 smithi032 ceph-mon[22403]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:35.508 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:35 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/3324378281' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-03T08:55:35.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:35 smithi123 ceph-mon[24496]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:35.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:35 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/3324378281' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-03T08:55:36.959 INFO:teuthology.orchestra.run.smithi032.stdout:111669149727 2024-05-03T08:55:37.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:37 smithi032 ceph-mon[22403]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:37.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:37 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/1406306211' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-03T08:55:37.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:37 smithi123 ceph-mon[24496]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:37.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:37 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/1406306211' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-03T08:55:38.213 INFO:teuthology.orchestra.run.smithi032.stdout:167503724567 2024-05-03T08:55:38.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:38 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/3452863748' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-03T08:55:38.798 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:38 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/3452863748' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-03T08:55:38.823 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462926 got 201863462929 for osd.6 2024-05-03T08:55:38.823 DEBUG:teuthology.parallel:result is None 2024-05-03T08:55:39.536 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:39 smithi032 ceph-mon[22403]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:39.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:39 smithi123 ceph-mon[24496]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:40.797 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149724 got 111669149727 for osd.3 2024-05-03T08:55:40.798 DEBUG:teuthology.parallel:result is None 2024-05-03T08:55:41.377 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724565 got 167503724567 for osd.5 2024-05-03T08:55:41.377 DEBUG:teuthology.parallel:result is None 2024-05-03T08:55:41.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:41 smithi032 ceph-mon[22403]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:41.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:41 smithi123 ceph-mon[24496]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:41.743 INFO:teuthology.orchestra.run.smithi032.stdout:38654705713 2024-05-03T08:55:42.432 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705710 got 38654705713 for osd.0 2024-05-03T08:55:42.432 DEBUG:teuthology.parallel:result is None 2024-05-03T08:55:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:42 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/2120182543' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-03T08:55:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:55:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:55:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:42 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/1695991194' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-03T08:55:42.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:42 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/2120182543' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-03T08:55:42.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:55:42.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:55:42.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:42 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/1695991194' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-03T08:55:42.792 INFO:teuthology.orchestra.run.smithi032.stdout:133143986203 2024-05-03T08:55:43.381 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986200 got 133143986203 for osd.4 2024-05-03T08:55:43.381 DEBUG:teuthology.parallel:result is None 2024-05-03T08:55:43.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:43 smithi032 ceph-mon[22403]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:43.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:43 smithi123 ceph-mon[24496]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:43.774 INFO:teuthology.orchestra.run.smithi032.stdout:60129542188 2024-05-03T08:55:43.990 INFO:teuthology.orchestra.run.smithi032.stdout:236223201293 2024-05-03T08:55:44.386 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542184 got 60129542188 for osd.1 2024-05-03T08:55:44.386 DEBUG:teuthology.parallel:result is None 2024-05-03T08:55:44.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:44 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/1459687456' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-03T08:55:44.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:44 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/3955904679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-03T08:55:44.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:44 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/1459687456' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-03T08:55:44.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:44 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/3955904679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-03T08:55:45.060 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201290 got 236223201293 for osd.7 2024-05-03T08:55:45.060 DEBUG:teuthology.parallel:result is None 2024-05-03T08:55:45.061 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-05-03T08:55:45.061 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph pg dump --format=json 2024-05-03T08:55:45.291 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:45.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:45 smithi123 ceph-mon[24496]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:45.750 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:45 smithi032 ceph-mon[22403]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:47.182 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:55:47.184 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-05-03T08:55:47.617 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:47 smithi032 ceph-mon[22403]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:47.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:47 smithi123 ceph-mon[24496]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:47.908 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":199,"stamp":"2024-05-03T08:55:45.774300+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48944,"kb_used_data":6320,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912272,"statfs":{"total":767960285184,"available":767910166528,"internally_reserved":0,"allocated":6471680,"data_stored":3548060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002452"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-05-03T08:54:51.303816+0000","last_change":"2024-05-03T08:54:51.303816+0000","last_active":"2024-05-03T08:54:51.303816+0000","last_peered":"2024-05-03T08:54:51.303816+0000","last_clean":"2024-05-03T08:54:51.303816+0000","last_became_active":"2024-05-03T08:54:51.303349+0000","last_became_peered":"2024-05-03T08:54:51.303349+0000","last_unstale":"2024-05-03T08:54:51.303816+0000","last_undegraded":"2024-05-03T08:54:51.303816+0000","last_fullsized":"2024-05-03T08:54:51.303816+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-03T08:52:48.234151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-03T08:52:48.234151+0000","last_clean_scrub_stamp":"2024-05-03T08:52:48.234151+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-04T14:48:39.099524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60799999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54800000000000004}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55300000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57099999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47599999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.497}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.625}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":1138688,"data_stored":773896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri May 3 08:55:23 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.442},{"interface":"front","average":{"1min":1.488,"5min":1.488,"15min":1.488},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":19.170000000000002,"5min":19.170000000000002,"15min":19.170000000000002},"last":0.56000000000000005}]},{"osd":1,"last update":"Fri May 3 08:55:23 2024","interfaces":[{"interface":"back","average":{"1min":1.5169999999999999,"5min":1.5169999999999999,"15min":1.5169999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":19.198,"5min":19.198,"15min":19.198},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.4730000000000001,"5min":1.4730000000000001,"15min":1.4730000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":19.181999999999999,"5min":19.181999999999999,"15min":19.181999999999999},"last":0.40699999999999997}]},{"osd":2,"last update":"Fri May 3 08:55:23 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.59599999999999997}]},{"osd":3,"last update":"Fri May 3 08:55:23 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.48899999999999999}]},{"osd":4,"last update":"Fri May 3 08:55:23 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.497},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.47999999999999998}]},{"osd":5,"last update":"Fri May 3 08:55:23 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.36799999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44700000000000001}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6364,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738788,"statfs":{"total":95995035648,"available":95988518912,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri May 3 08:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.35299999999999998,"15min":0.33600000000000002},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.38100000000000001,"15min":0.34699999999999998},"min":{"1min":0.28100000000000003,"5min":0.246,"15min":0.246},"max":{"1min":1.2689999999999999,"5min":1.2689999999999999,"15min":1.2689999999999999},"last":0.31}]},{"osd":2,"last update":"Fri May 3 08:54:50 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.41199999999999998,"15min":0.40400000000000003},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.441,"5min":0.38500000000000001,"15min":0.376},"min":{"1min":0.27400000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.34399999999999997}]},{"osd":3,"last update":"Fri May 3 08:55:21 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.45600000000000002,"15min":0.45000000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.6319999999999999,"5min":1.6319999999999999,"15min":1.6319999999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.49399999999999999,"15min":0.48799999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":1.502,"5min":1.502,"15min":1.502},"last":0.40200000000000002}]},{"osd":4,"last update":"Fri May 3 08:54:39 2024","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":10.090999999999999}]},{"osd":5,"last update":"Fri May 3 08:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.53400000000000003}]},{"osd":6,"last update":"Fri May 3 08:55:26 2024","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":1.377,"15min":1.377},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":12.478999999999999,"5min":12.478999999999999,"15min":12.478999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.383,"5min":1.383,"15min":1.383},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":12.452999999999999,"5min":12.452999999999999,"15min":12.452999999999999},"last":0.56200000000000006}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":10.067}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri May 3 08:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.36099999999999999,"15min":0.35099999999999998},"min":{"1min":0.26600000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.38400000000000001,"15min":0.36099999999999999},"min":{"1min":0.248,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.51800000000000002}]},{"osd":2,"last update":"Fri May 3 08:54:49 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.433,"15min":0.42599999999999999},"min":{"1min":0.35199999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.441,"5min":0.41399999999999998,"15min":0.40999999999999998},"min":{"1min":0.31,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.54000000000000004}]},{"osd":3,"last update":"Fri May 3 08:55:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.495,"15min":0.48899999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.6599999999999999,"5min":1.6599999999999999,"15min":1.6599999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.48999999999999999,"15min":0.48499999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":1.593,"5min":1.593,"15min":1.593},"last":0.501}]},{"osd":4,"last update":"Fri May 3 08:55:40 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.59199999999999997,"15min":0.58199999999999996},"min":{"1min":0.44600000000000001,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.208,"5min":1.208,"15min":1.208},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.621,"5min":0.58399999999999996,"15min":0.57799999999999996},"min":{"1min":0.42299999999999999,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.76500000000000001}]},{"osd":5,"last update":"Fri May 3 08:54:59 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.75}]},{"osd":6,"last update":"Fri May 3 08:55:22 2024","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.2669999999999999},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":12.579000000000001,"5min":12.579000000000001,"15min":12.579000000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":1.2889999999999999,"5min":1.2889999999999999,"15min":1.2889999999999999},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":12.750999999999999,"5min":12.750999999999999,"15min":12.750999999999999},"last":0.69799999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65800000000000003}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6300,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738852,"statfs":{"total":95995035648,"available":95988584448,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri May 3 08:54:57 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.36799999999999999,"15min":0.35799999999999998},"min":{"1min":0.28199999999999997,"5min":0.253,"15min":0.253},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.38400000000000001,"15min":0.372},"min":{"1min":0.28399999999999997,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.58299999999999996}]},{"osd":1,"last update":"Fri May 3 08:54:57 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.38400000000000001,"15min":0.376},"min":{"1min":0.35199999999999998,"5min":0.245,"15min":0.245},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.40799999999999997,"15min":0.39600000000000002},"min":{"1min":0.312,"5min":0.222,"15min":0.222},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.56899999999999995}]},{"osd":3,"last update":"Fri May 3 08:55:14 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.499,"15min":0.48299999999999998},"min":{"1min":0.39400000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.9079999999999999,"5min":1.9079999999999999,"15min":1.9079999999999999},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.47599999999999998,"15min":0.46999999999999997},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.2210000000000001,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":0.53500000000000003}]},{"osd":4,"last update":"Fri May 3 08:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":0.66400000000000003,"15min":0.63700000000000001},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":3.1160000000000001,"5min":3.1160000000000001,"15min":3.1160000000000001},"last":3.1160000000000001},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.68400000000000005,"15min":0.65600000000000003},"min":{"1min":0.50700000000000001,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":3.1059999999999999,"5min":3.1059999999999999,"15min":3.1059999999999999},"last":3.1059999999999999}]},{"osd":5,"last update":"Fri May 3 08:55:01 2024","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.137,"5min":1.137,"15min":1.137},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.1279999999999999,"5min":1.1279999999999999,"15min":1.1279999999999999},"last":3.1309999999999998}]},{"osd":6,"last update":"Fri May 3 08:55:27 2024","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":1.95,"5min":1.95,"15min":1.95},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.1519999999999999,"5min":1.1519999999999999,"15min":1.1519999999999999},"last":0.76300000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.1659999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri May 3 08:55:19 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.42699999999999999,"15min":0.42799999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.58499999999999996,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.436,"15min":0.437},"min":{"1min":0.33600000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.36499999999999999}]},{"osd":1,"last update":"Fri May 3 08:55:19 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.36699999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.56100000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.58199999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.32600000000000001}]},{"osd":2,"last update":"Fri May 3 08:55:19 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.48699999999999999,"15min":0.49199999999999999},"min":{"1min":0.32000000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.55200000000000005,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.378},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.47999999999999998,"15min":0.48899999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.60099999999999998,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.311}]},{"osd":4,"last update":"Fri May 3 08:55:42 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.59999999999999998,"15min":0.59799999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.60299999999999998,"15min":0.59699999999999998},"min":{"1min":0.499,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.53600000000000003}]},{"osd":5,"last update":"Fri May 3 08:54:59 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.72799999999999998}]},{"osd":6,"last update":"Fri May 3 08:55:28 2024","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.0089999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.55100000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57099999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986205,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5788,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739364,"statfs":{"total":95995035648,"available":95989108736,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri May 3 08:55:40 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.56399999999999995,"15min":0.56999999999999995},"min":{"1min":0.374,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.59299999999999997,"15min":0.59099999999999997},"min":{"1min":0.46700000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.59399999999999997}]},{"osd":1,"last update":"Fri May 3 08:55:40 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.54400000000000004,"15min":0.54200000000000004},"min":{"1min":0.36299999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.55300000000000005,"15min":0.54900000000000004},"min":{"1min":0.376,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.72299999999999998,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.45700000000000002}]},{"osd":2,"last update":"Fri May 3 08:55:40 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55800000000000005,"15min":0.55900000000000005},"min":{"1min":0.38600000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.58399999999999996,"15min":0.58799999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.51100000000000001}]},{"osd":3,"last update":"Fri May 3 08:55:40 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.39700000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.72099999999999997,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.59099999999999997,"15min":0.58899999999999997},"min":{"1min":0.44500000000000001,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.52100000000000002}]},{"osd":5,"last update":"Fri May 3 08:55:03 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.54200000000000004}]},{"osd":6,"last update":"Fri May 3 08:55:27 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.438}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56699999999999995}]}]},{"osd":5,"up_from":39,"seq":167503724569,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5660,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739492,"statfs":{"total":95995035648,"available":95989239808,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri May 3 08:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.55500000000000005}]},{"osd":1,"last update":"Fri May 3 08:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.46200000000000002}]},{"osd":2,"last update":"Fri May 3 08:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.67600000000000005}]},{"osd":3,"last update":"Fri May 3 08:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.73099999999999998}]},{"osd":4,"last update":"Fri May 3 08:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.68999999999999995}]},{"osd":6,"last update":"Fri May 3 08:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.746}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78100000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-03T08:55:47.910 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph pg dump --format=json 2024-05-03T08:55:48.136 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:48.549 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:48 smithi032 ceph-mon[22403]: from='client.14448 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-03T08:55:48.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:48 smithi123 ceph-mon[24496]: from='client.14448 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-03T08:55:49.547 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:49 smithi032 ceph-mon[22403]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:49.547 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:55:49.548 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:55:49.548 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:55:49.548 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:49 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:55:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:49 smithi123 ceph-mon[24496]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:55:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:55:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:55:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:49 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:55:49.941 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:55:49.943 INFO:teuthology.orchestra.run.smithi032.stderr:dumped all 2024-05-03T08:55:50.631 INFO:teuthology.orchestra.run.smithi032.stdout:{"pg_ready":true,"pg_map":{"version":201,"stamp":"2024-05-03T08:55:49.775015+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48944,"kb_used_data":6320,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912272,"statfs":{"total":767960285184,"available":767910166528,"internally_reserved":0,"allocated":6471680,"data_stored":3548060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002160"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-05-03T08:54:51.303816+0000","last_change":"2024-05-03T08:54:51.303816+0000","last_active":"2024-05-03T08:54:51.303816+0000","last_peered":"2024-05-03T08:54:51.303816+0000","last_clean":"2024-05-03T08:54:51.303816+0000","last_became_active":"2024-05-03T08:54:51.303349+0000","last_became_peered":"2024-05-03T08:54:51.303349+0000","last_unstale":"2024-05-03T08:54:51.303816+0000","last_undegraded":"2024-05-03T08:54:51.303816+0000","last_fullsized":"2024-05-03T08:54:51.303816+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-03T08:52:48.234151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-03T08:52:48.234151+0000","last_clean_scrub_stamp":"2024-05-03T08:52:48.234151+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-04T14:48:39.099524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201295,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54100000000000004}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59899999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57399999999999995}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.623}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39100000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41199999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47899999999999998}]}]},{"osd":6,"up_from":47,"seq":201863462932,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":1138688,"data_stored":773896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri May 3 08:55:23 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.488,"5min":1.488,"15min":1.488},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":19.170000000000002,"5min":19.170000000000002,"15min":19.170000000000002},"last":0.47199999999999998}]},{"osd":1,"last update":"Fri May 3 08:55:23 2024","interfaces":[{"interface":"back","average":{"1min":1.5169999999999999,"5min":1.5169999999999999,"15min":1.5169999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":19.198,"5min":19.198,"15min":19.198},"last":0.371},{"interface":"front","average":{"1min":1.4730000000000001,"5min":1.4730000000000001,"15min":1.4730000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":19.181999999999999,"5min":19.181999999999999,"15min":19.181999999999999},"last":0.38900000000000001}]},{"osd":2,"last update":"Fri May 3 08:55:23 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.35399999999999998}]},{"osd":3,"last update":"Fri May 3 08:55:23 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.442}]},{"osd":4,"last update":"Fri May 3 08:55:23 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.503},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.33000000000000002}]},{"osd":5,"last update":"Fri May 3 08:55:23 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.45500000000000002}]},{"osd":7,"last update":"Fri May 3 08:55:47 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.51800000000000002}]}]},{"osd":1,"up_from":14,"seq":60129542189,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6364,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738788,"statfs":{"total":95995035648,"available":95988518912,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri May 3 08:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.35299999999999998,"15min":0.33600000000000002},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.317},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.38100000000000001,"15min":0.34699999999999998},"min":{"1min":0.28100000000000003,"5min":0.246,"15min":0.246},"max":{"1min":1.2689999999999999,"5min":1.2689999999999999,"15min":1.2689999999999999},"last":0.33300000000000002}]},{"osd":2,"last update":"Fri May 3 08:54:50 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.41199999999999998,"15min":0.40400000000000003},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.441,"5min":0.38500000000000001,"15min":0.376},"min":{"1min":0.27400000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.29699999999999999}]},{"osd":3,"last update":"Fri May 3 08:55:21 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.45600000000000002,"15min":0.45000000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.6319999999999999,"5min":1.6319999999999999,"15min":1.6319999999999999},"last":0.375},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.49399999999999999,"15min":0.48799999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":1.502,"5min":1.502,"15min":1.502},"last":0.39900000000000002}]},{"osd":4,"last update":"Fri May 3 08:55:45 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.64100000000000001,"15min":0.64400000000000002},"min":{"1min":0.40999999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.976,"5min":0.90100000000000002,"15min":0.72199999999999998},"min":{"1min":0.48199999999999998,"5min":0.432,"15min":0.432},"max":{"1min":12.436999999999999,"5min":12.436999999999999,"15min":12.436999999999999},"last":0.68100000000000005}]},{"osd":5,"last update":"Fri May 3 08:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.71899999999999997}]},{"osd":6,"last update":"Fri May 3 08:55:26 2024","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":1.377,"15min":1.377},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":12.478999999999999,"5min":12.478999999999999,"15min":12.478999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.383,"5min":1.383,"15min":1.383},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":12.452999999999999,"5min":12.452999999999999,"15min":12.452999999999999},"last":0.66900000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55600000000000005}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri May 3 08:55:22 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.36099999999999999,"15min":0.35099999999999998},"min":{"1min":0.26600000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.38400000000000001,"15min":0.36099999999999999},"min":{"1min":0.248,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.51800000000000002}]},{"osd":2,"last update":"Fri May 3 08:54:49 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.433,"15min":0.42599999999999999},"min":{"1min":0.35199999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.441,"5min":0.41399999999999998,"15min":0.40999999999999998},"min":{"1min":0.31,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.54000000000000004}]},{"osd":3,"last update":"Fri May 3 08:55:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.495,"15min":0.48899999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.6599999999999999,"5min":1.6599999999999999,"15min":1.6599999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.48999999999999999,"15min":0.48499999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":1.593,"5min":1.593,"15min":1.593},"last":0.501}]},{"osd":4,"last update":"Fri May 3 08:55:40 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.59199999999999997,"15min":0.58199999999999996},"min":{"1min":0.44600000000000001,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.208,"5min":1.208,"15min":1.208},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.621,"5min":0.58399999999999996,"15min":0.57799999999999996},"min":{"1min":0.42299999999999999,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.76500000000000001}]},{"osd":5,"last update":"Fri May 3 08:54:59 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.75}]},{"osd":6,"last update":"Fri May 3 08:55:22 2024","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.2669999999999999},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":12.579000000000001,"5min":12.579000000000001,"15min":12.579000000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":1.2889999999999999,"5min":1.2889999999999999,"15min":1.2889999999999999},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":12.750999999999999,"5min":12.750999999999999,"15min":12.750999999999999},"last":0.69799999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65800000000000003}]}]},{"osd":2,"up_from":19,"seq":81604378664,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6300,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738852,"statfs":{"total":95995035648,"available":95988584448,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri May 3 08:54:57 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.36799999999999999,"15min":0.35799999999999998},"min":{"1min":0.28199999999999997,"5min":0.253,"15min":0.253},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.254},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.38400000000000001,"15min":0.372},"min":{"1min":0.28399999999999997,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.308}]},{"osd":1,"last update":"Fri May 3 08:54:57 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.38400000000000001,"15min":0.376},"min":{"1min":0.35199999999999998,"5min":0.245,"15min":0.245},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.40799999999999997,"15min":0.39600000000000002},"min":{"1min":0.312,"5min":0.222,"15min":0.222},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.29799999999999999}]},{"osd":3,"last update":"Fri May 3 08:55:14 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.499,"15min":0.48299999999999998},"min":{"1min":0.39400000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.9079999999999999,"5min":1.9079999999999999,"15min":1.9079999999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.47599999999999998,"15min":0.46999999999999997},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.2210000000000001,"5min":1.2210000000000001,"15min":1.2210000000000001},"last":0.378}]},{"osd":4,"last update":"Fri May 3 08:55:37 2024","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":0.66400000000000003,"15min":0.63700000000000001},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":3.1160000000000001,"5min":3.1160000000000001,"15min":3.1160000000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.68400000000000005,"15min":0.65600000000000003},"min":{"1min":0.50700000000000001,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":3.1059999999999999,"5min":3.1059999999999999,"15min":3.1059999999999999},"last":0.48199999999999998}]},{"osd":5,"last update":"Fri May 3 08:55:01 2024","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.137,"5min":1.137,"15min":1.137},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.1279999999999999,"5min":1.1279999999999999,"15min":1.1279999999999999},"last":0.46999999999999997}]},{"osd":6,"last update":"Fri May 3 08:55:27 2024","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":1.95,"5min":1.95,"15min":1.95},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.1519999999999999,"5min":1.1519999999999999,"15min":1.1519999999999999},"last":0.438}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.505}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri May 3 08:55:19 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.42699999999999999,"15min":0.42799999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.58499999999999996,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.436,"15min":0.437},"min":{"1min":0.33600000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.36499999999999999}]},{"osd":1,"last update":"Fri May 3 08:55:19 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.36699999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.56100000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.58199999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.32600000000000001}]},{"osd":2,"last update":"Fri May 3 08:55:19 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.48699999999999999,"15min":0.49199999999999999},"min":{"1min":0.32000000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.55200000000000005,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.378},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.47999999999999998,"15min":0.48899999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.60099999999999998,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.311}]},{"osd":4,"last update":"Fri May 3 08:55:42 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.59999999999999998,"15min":0.59799999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.60299999999999998,"15min":0.59699999999999998},"min":{"1min":0.499,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.53600000000000003}]},{"osd":5,"last update":"Fri May 3 08:54:59 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.72799999999999998}]},{"osd":6,"last update":"Fri May 3 08:55:28 2024","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.0089999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.55100000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57099999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986205,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5788,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739364,"statfs":{"total":95995035648,"available":95989108736,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri May 3 08:55:40 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.56399999999999995,"15min":0.56999999999999995},"min":{"1min":0.374,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.59299999999999997,"15min":0.59099999999999997},"min":{"1min":0.46700000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.59399999999999997}]},{"osd":1,"last update":"Fri May 3 08:55:40 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.54400000000000004,"15min":0.54200000000000004},"min":{"1min":0.36299999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.55300000000000005,"15min":0.54900000000000004},"min":{"1min":0.376,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.72299999999999998,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.45700000000000002}]},{"osd":2,"last update":"Fri May 3 08:55:40 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55800000000000005,"15min":0.55900000000000005},"min":{"1min":0.38600000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.58399999999999996,"15min":0.58799999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.51100000000000001}]},{"osd":3,"last update":"Fri May 3 08:55:40 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.39700000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.72099999999999997,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.59099999999999997,"15min":0.58899999999999997},"min":{"1min":0.44500000000000001,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.52100000000000002}]},{"osd":5,"last update":"Fri May 3 08:55:03 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.54200000000000004}]},{"osd":6,"last update":"Fri May 3 08:55:27 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.438}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56699999999999995}]}]},{"osd":5,"up_from":39,"seq":167503724570,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5660,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739492,"statfs":{"total":95995035648,"available":95989239808,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri May 3 08:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.52100000000000002}]},{"osd":1,"last update":"Fri May 3 08:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.45500000000000002}]},{"osd":2,"last update":"Fri May 3 08:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.432},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.48199999999999998}]},{"osd":3,"last update":"Fri May 3 08:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.53700000000000003}]},{"osd":4,"last update":"Fri May 3 08:54:58 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.50600000000000001}]},{"osd":6,"last update":"Fri May 3 08:55:26 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.58199999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55400000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-03T08:55:50.632 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-05-03T08:55:50.633 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-05-03T08:55:50.633 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-05-03T08:55:50.633 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph health --format=json 2024-05-03T08:55:50.870 INFO:teuthology.orchestra.run.smithi032.stderr:Inferring config /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/mon.smithi032/config 2024-05-03T08:55:51.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:51 smithi032 ceph-mon[22403]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:51.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:51 smithi032 ceph-mon[22403]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-03T08:55:51.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:51 smithi123 ceph-mon[24496]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:51.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:51 smithi123 ceph-mon[24496]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-03T08:55:52.732 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:55:52.732 INFO:teuthology.orchestra.run.smithi032.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-05-03T08:55:53.368 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-05-03T08:55:53.368 INFO:tasks.cephadm:Setup complete, yielding 2024-05-03T08:55:53.368 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-03T08:55:53.382 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi032.front.sepia.ceph.com 2024-05-03T08:55:53.382 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- bash -c 'ceph orch status' 2024-05-03T08:55:53.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:53 smithi032 ceph-mon[22403]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:53.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:53 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/763974571' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-03T08:55:53.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:53 smithi123 ceph-mon[24496]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:53.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:53 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/763974571' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-03T08:55:55.466 INFO:teuthology.orchestra.run.smithi032.stdout:Backend: cephadm 2024-05-03T08:55:55.466 INFO:teuthology.orchestra.run.smithi032.stdout:Available: Yes 2024-05-03T08:55:55.466 INFO:teuthology.orchestra.run.smithi032.stdout:Paused: No 2024-05-03T08:55:55.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:55 smithi123 ceph-mon[24496]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:55.839 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:55 smithi032 ceph-mon[22403]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:56.132 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- bash -c 'ceph orch ps' 2024-05-03T08:55:56.709 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:56 smithi032 ceph-mon[22403]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:55:56.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:56 smithi123 ceph-mon[24496]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:55:57.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:57 smithi123 ceph-mon[24496]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:57.728 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:57 smithi032 ceph-mon[22403]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:58.134 INFO:teuthology.orchestra.run.smithi032.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-03T08:55:58.135 INFO:teuthology.orchestra.run.smithi032.stdout:alertmanager.smithi032 smithi032 *:9093,9094 running (4m) 2m ago 5m 14.6M - ba2b418f427c ddf53d5d27fc 2024-05-03T08:55:58.135 INFO:teuthology.orchestra.run.smithi032.stdout:crash.smithi032 smithi032 running (5m) 2m ago 5m 6409k - 17.2.0 e1d6a67b021e 38dc8227bd42 2024-05-03T08:55:58.135 INFO:teuthology.orchestra.run.smithi032.stdout:crash.smithi123 smithi123 running (5m) 72s ago 5m 6405k - 17.2.0 e1d6a67b021e 2bce696b8db8 2024-05-03T08:55:58.135 INFO:teuthology.orchestra.run.smithi032.stdout:grafana.smithi032 smithi032 *:3000 running (4m) 2m ago 5m 43.5M - 8.3.5 dad864ee21e9 81ca1cc63b54 2024-05-03T08:55:58.135 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.smithi032.umllay smithi032 *:9283 running (7m) 2m ago 6m 448M - 17.2.0 e1d6a67b021e 537e53955aba 2024-05-03T08:55:58.135 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.smithi123.xrxinh smithi123 *:8443,9283 running (5m) 72s ago 5m 411M - 17.2.0 e1d6a67b021e e7499f622d47 2024-05-03T08:55:58.135 INFO:teuthology.orchestra.run.smithi032.stdout:mon.smithi032 smithi032 running (7m) 2m ago 7m 42.6M 2048M 17.2.0 e1d6a67b021e 7e5da84a501e 2024-05-03T08:55:58.135 INFO:teuthology.orchestra.run.smithi032.stdout:mon.smithi123 smithi123 running (4m) 72s ago 4m 42.0M 2048M 17.2.0 e1d6a67b021e 3a32332bb758 2024-05-03T08:55:58.135 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.smithi032 smithi032 *:9100 running (5m) 2m ago 5m 17.1M - 1dbe0e931976 ae21c41038e9 2024-05-03T08:55:58.135 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.smithi123 smithi123 *:9100 running (4m) 72s ago 4m 15.8M - 1dbe0e931976 dabbb678f527 2024-05-03T08:55:58.135 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 smithi032 running (4m) 2m ago 4m 33.0M 3217M 17.2.0 e1d6a67b021e 167471b12229 2024-05-03T08:55:58.135 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 smithi032 running (3m) 2m ago 3m 37.5M 3217M 17.2.0 e1d6a67b021e 21b9c0b15d6f 2024-05-03T08:55:58.135 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 smithi032 running (3m) 2m ago 3m 30.4M 3217M 17.2.0 e1d6a67b021e 6dfd926898e3 2024-05-03T08:55:58.136 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 smithi032 running (2m) 2m ago 2m 11.0M 3217M 17.2.0 e1d6a67b021e 947099fbe826 2024-05-03T08:55:58.136 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 smithi123 running (2m) 72s ago 2m 34.0M 3984M 17.2.0 e1d6a67b021e 150ddc5685f9 2024-05-03T08:55:58.136 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 smithi123 running (2m) 72s ago 2m 35.7M 3984M 17.2.0 e1d6a67b021e 72ba92ce4b53 2024-05-03T08:55:58.136 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 smithi123 running (101s) 72s ago 101s 33.4M 3984M 17.2.0 e1d6a67b021e 2bc752f6bfc5 2024-05-03T08:55:58.136 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 smithi123 running (77s) 72s ago 77s 13.0M 3984M 17.2.0 e1d6a67b021e 40d54616bfbe 2024-05-03T08:55:58.136 INFO:teuthology.orchestra.run.smithi032.stdout:prometheus.smithi032 smithi032 *:9095 running (4m) 2m ago 4m 53.8M - 514e6a882f6e 6763db48c356 2024-05-03T08:55:58.780 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- bash -c 'ceph orch ls' 2024-05-03T08:55:59.677 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:59 smithi032 ceph-mon[22403]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:59.677 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:55:59 smithi032 ceph-mon[22403]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:55:59.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:59 smithi123 ceph-mon[24496]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:55:59.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:55:59 smithi123 ceph-mon[24496]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:00.707 INFO:teuthology.orchestra.run.smithi032.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-05-03T08:56:00.707 INFO:teuthology.orchestra.run.smithi032.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-05-03T08:56:00.707 INFO:teuthology.orchestra.run.smithi032.stdout:crash 2/2 2m ago 6m * 2024-05-03T08:56:00.707 INFO:teuthology.orchestra.run.smithi032.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-05-03T08:56:00.707 INFO:teuthology.orchestra.run.smithi032.stdout:mgr 2/2 2m ago 6m count:2 2024-05-03T08:56:00.707 INFO:teuthology.orchestra.run.smithi032.stdout:mon 2/2 2m ago 5m smithi032:172.21.15.32=smithi032;smithi123:172.21.15.123=smithi123;count:2 2024-05-03T08:56:00.707 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-05-03T08:56:00.708 INFO:teuthology.orchestra.run.smithi032.stdout:osd 8 2m ago - 2024-05-03T08:56:00.708 INFO:teuthology.orchestra.run.smithi032.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-05-03T08:56:01.301 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- bash -c 'ceph orch host ls' 2024-05-03T08:56:01.612 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:01 smithi032 ceph-mon[22403]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:01.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:01 smithi123 ceph-mon[24496]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:02.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:02 smithi032 ceph-mon[22403]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:02.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:02 smithi123 ceph-mon[24496]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:03.385 INFO:teuthology.orchestra.run.smithi032.stdout:HOST ADDR LABELS STATUS 2024-05-03T08:56:03.385 INFO:teuthology.orchestra.run.smithi032.stdout:smithi032 172.21.15.32 2024-05-03T08:56:03.385 INFO:teuthology.orchestra.run.smithi032.stdout:smithi123 172.21.15.123 2024-05-03T08:56:03.385 INFO:teuthology.orchestra.run.smithi032.stdout:2 hosts in cluster 2024-05-03T08:56:03.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:03 smithi032 ceph-mon[22403]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:03.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:03 smithi123 ceph-mon[24496]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:04.024 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- bash -c 'ceph orch device ls' 2024-05-03T08:56:04.601 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:04 smithi032 ceph-mon[22403]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:04.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:04 smithi123 ceph-mon[24496]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:05.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:05 smithi032 ceph-mon[22403]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:05.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:05 smithi123 ceph-mon[24496]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:06.060 INFO:teuthology.orchestra.run.smithi032.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-05-03T08:56:06.060 INFO:teuthology.orchestra.run.smithi032.stdout:smithi032 /dev/nvme0n1 ssd INTEL_SSDPED1D480GA_PHMB747402DN480DGN 480G LVM detected, locked 2024-05-03T08:56:06.060 INFO:teuthology.orchestra.run.smithi032.stdout:smithi123 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400YK400BGN 400G LVM detected, locked 2024-05-03T08:56:06.810 INFO:teuthology.run_tasks:Running task vip.exec... 2024-05-03T08:56:06.823 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi032.front.sepia.ceph.com 2024-05-03T08:56:06.823 DEBUG:teuthology.orchestra.run.smithi032:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-05-03T08:56:06.852 INFO:teuthology.orchestra.run.smithi032.stderr:+ systemctl stop nfs-server 2024-05-03T08:56:06.860 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi123.front.sepia.ceph.com 2024-05-03T08:56:06.860 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-05-03T08:56:06.889 INFO:teuthology.orchestra.run.smithi123.stderr:+ systemctl stop nfs-server 2024-05-03T08:56:06.897 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-03T08:56:06.906 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi032.front.sepia.ceph.com 2024-05-03T08:56:06.906 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-05-03T08:56:07.509 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:07 smithi032 ceph-mon[22403]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:07.509 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:07 smithi032 ceph-mon[22403]: from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:07.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:07 smithi123 ceph-mon[24496]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:07.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:07 smithi123 ceph-mon[24496]: from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:10.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:09 smithi123 ceph-mon[24496]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:10.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:09 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-05-03T08:56:10.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:09 smithi032 ceph-mon[22403]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:10.388 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:09 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-05-03T08:56:10.828 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi032[22399]: 2024-05-03T08:56:10.469+0000 7f5920942700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-05-03T08:56:11.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: from='client.14480 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:11.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-05-03T08:56:11.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: osdmap e60: 8 total, 8 up, 8 in 2024-05-03T08:56:11.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-05-03T08:56:11.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: pgmap v212: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:11.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-05-03T08:56:11.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: osdmap e61: 8 total, 8 up, 8 in 2024-05-03T08:56:11.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-05-03T08:56:11.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-05-03T08:56:11.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-05-03T08:56:11.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: osdmap e62: 8 total, 8 up, 8 in 2024-05-03T08:56:11.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-05-03T08:56:11.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: fsmap foofs:0 2024-05-03T08:56:11.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: Saving service mds.foofs spec with placement count:2 2024-05-03T08:56:11.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:11.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:56:11.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:11.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:56:11.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:11.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:11.140 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi032.axmmtz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-03T08:56:11.140 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi032.axmmtz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-03T08:56:11.140 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:10 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:11.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: from='client.14480 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:11.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-05-03T08:56:11.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: osdmap e60: 8 total, 8 up, 8 in 2024-05-03T08:56:11.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-05-03T08:56:11.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: pgmap v212: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:11.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-05-03T08:56:11.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: osdmap e61: 8 total, 8 up, 8 in 2024-05-03T08:56:11.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-05-03T08:56:11.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-05-03T08:56:11.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-05-03T08:56:11.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: osdmap e62: 8 total, 8 up, 8 in 2024-05-03T08:56:11.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-05-03T08:56:11.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: fsmap foofs:0 2024-05-03T08:56:11.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: Saving service mds.foofs spec with placement count:2 2024-05-03T08:56:11.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:11.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:56:11.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:11.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:56:11.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:11.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:11.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi032.axmmtz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-03T08:56:11.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi032.axmmtz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-03T08:56:11.229 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:10 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:11.299 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-05-03T08:56:11.308 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-05-03T08:56:11.309 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph orch ls -f json 2024-05-03T08:56:12.840 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:12 smithi032 ceph-mon[22403]: Deploying daemon mds.foofs.smithi032.axmmtz on smithi032 2024-05-03T08:56:12.840 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:12 smithi032 ceph-mon[22403]: osdmap e63: 8 total, 8 up, 8 in 2024-05-03T08:56:12.840 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:12 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:12.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:12 smithi123 ceph-mon[24496]: Deploying daemon mds.foofs.smithi032.axmmtz on smithi032 2024-05-03T08:56:12.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:12 smithi123 ceph-mon[24496]: osdmap e63: 8 total, 8 up, 8 in 2024-05-03T08:56:12.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:12 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:13.853 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:13 smithi032 ceph-mon[22403]: pgmap v216: 65 pgs: 10 creating+peering, 12 active+clean, 43 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:13.853 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:13 smithi032 ceph-mon[22403]: osdmap e64: 8 total, 8 up, 8 in 2024-05-03T08:56:13.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:13 smithi123 ceph-mon[24496]: pgmap v216: 65 pgs: 10 creating+peering, 12 active+clean, 43 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:13.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:13 smithi123 ceph-mon[24496]: osdmap e64: 8 total, 8 up, 8 in 2024-05-03T08:56:14.625 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:56:14.625 INFO:teuthology.orchestra.run.smithi032.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-05-03T08:49:37.936309Z", "last_refresh": "2024-05-03T08:53:14.571967Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-05-03T08:49:31.820376Z", "last_refresh": "2024-05-03T08:53:14.572037Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-05-03T08:49:34.601976Z", "last_refresh": "2024-05-03T08:53:14.572102Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-05-03T08:56:10.489250Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-05-03T08:56:10.483051Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-05-03T08:49:29.676178Z", "last_refresh": "2024-05-03T08:53:14.571888Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi032:172.21.15.32=smithi032", "smithi123:172.21.15.123=smithi123"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-05-03T08:50:21.959405Z", "last_refresh": "2024-05-03T08:53:14.571732Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-05-03T08:49:36.655612Z", "last_refresh": "2024-05-03T08:53:14.572166Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-05-03T08:53:14.572295Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-05-03T08:49:33.325354Z", "last_refresh": "2024-05-03T08:53:14.572231Z", "ports": [9095], "running": 1, "size": 1}}] 2024-05-03T08:56:15.429 INFO:tasks.cephadm:mds.foofs has 0/2 2024-05-03T08:56:15.663 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:15 smithi032 ceph-mon[22403]: pgmap v218: 65 pgs: 10 creating+peering, 35 active+clean, 20 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:15.744 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:15 smithi123 ceph-mon[24496]: pgmap v218: 65 pgs: 10 creating+peering, 35 active+clean, 20 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:16.430 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph orch ls -f json 2024-05-03T08:56:16.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:16 smithi123 ceph-mon[24496]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-03T08:56:16.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:16 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:16.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:16 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.gtrlts", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-03T08:56:16.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:16 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.gtrlts", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-03T08:56:16.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:16 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:16.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:16 smithi123 ceph-mon[24496]: daemon mds.foofs.smithi032.axmmtz assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-05-03T08:56:16.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:16 smithi123 ceph-mon[24496]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-05-03T08:56:16.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:16 smithi123 ceph-mon[24496]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-05-03T08:56:16.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:16 smithi123 ceph-mon[24496]: Cluster is now healthy 2024-05-03T08:56:16.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:16 smithi123 ceph-mon[24496]: mds.? [v2:172.21.15.32:6834/1439645756,v1:172.21.15.32:6835/1439645756] up:boot 2024-05-03T08:56:16.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:16 smithi123 ceph-mon[24496]: fsmap foofs:1 {0=foofs.smithi032.axmmtz=up:creating} 2024-05-03T08:56:16.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:16 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi032.axmmtz"}]: dispatch 2024-05-03T08:56:16.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:16 smithi123 ceph-mon[24496]: daemon mds.foofs.smithi032.axmmtz is now active in filesystem foofs as rank 0 2024-05-03T08:56:16.740 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:16 smithi032 ceph-mon[22403]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-03T08:56:16.740 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:16 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:16.740 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:16 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.gtrlts", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-03T08:56:16.740 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:16 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.gtrlts", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-03T08:56:16.740 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:16 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:16.740 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:16 smithi032 ceph-mon[22403]: daemon mds.foofs.smithi032.axmmtz assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-05-03T08:56:16.740 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:16 smithi032 ceph-mon[22403]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-05-03T08:56:16.741 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:16 smithi032 ceph-mon[22403]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-05-03T08:56:16.741 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:16 smithi032 ceph-mon[22403]: Cluster is now healthy 2024-05-03T08:56:16.741 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:16 smithi032 ceph-mon[22403]: mds.? [v2:172.21.15.32:6834/1439645756,v1:172.21.15.32:6835/1439645756] up:boot 2024-05-03T08:56:16.741 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:16 smithi032 ceph-mon[22403]: fsmap foofs:1 {0=foofs.smithi032.axmmtz=up:creating} 2024-05-03T08:56:16.741 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:16 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi032.axmmtz"}]: dispatch 2024-05-03T08:56:16.741 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:16 smithi032 ceph-mon[22403]: daemon mds.foofs.smithi032.axmmtz is now active in filesystem foofs as rank 0 2024-05-03T08:56:17.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:17 smithi123 ceph-mon[24496]: Deploying daemon mds.foofs.smithi123.gtrlts on smithi123 2024-05-03T08:56:17.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:17 smithi123 ceph-mon[24496]: pgmap v219: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:17.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:17 smithi123 ceph-mon[24496]: mds.? [v2:172.21.15.32:6834/1439645756,v1:172.21.15.32:6835/1439645756] up:active 2024-05-03T08:56:17.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:17 smithi123 ceph-mon[24496]: fsmap foofs:1 {0=foofs.smithi032.axmmtz=up:active} 2024-05-03T08:56:17.789 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:17 smithi032 ceph-mon[22403]: Deploying daemon mds.foofs.smithi123.gtrlts on smithi123 2024-05-03T08:56:17.789 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:17 smithi032 ceph-mon[22403]: pgmap v219: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:17.789 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:17 smithi032 ceph-mon[22403]: mds.? [v2:172.21.15.32:6834/1439645756,v1:172.21.15.32:6835/1439645756] up:active 2024-05-03T08:56:17.789 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:17 smithi032 ceph-mon[22403]: fsmap foofs:1 {0=foofs.smithi032.axmmtz=up:active} 2024-05-03T08:56:18.463 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:56:18.463 INFO:teuthology.orchestra.run.smithi032.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-05-03T08:49:37.936309Z", "last_refresh": "2024-05-03T08:53:14.571967Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-05-03T08:49:31.820376Z", "last_refresh": "2024-05-03T08:53:14.572037Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-05-03T08:49:34.601976Z", "last_refresh": "2024-05-03T08:53:14.572102Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-05-03T08:56:10.489250Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-05-03T08:56:10.483051Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-05-03T08:49:29.676178Z", "last_refresh": "2024-05-03T08:53:14.571888Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi032:172.21.15.32=smithi032", "smithi123:172.21.15.123=smithi123"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-05-03T08:50:21.959405Z", "last_refresh": "2024-05-03T08:53:14.571732Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-05-03T08:49:36.655612Z", "last_refresh": "2024-05-03T08:53:14.572166Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-05-03T08:53:14.572295Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-05-03T08:49:33.325354Z", "last_refresh": "2024-05-03T08:53:14.572231Z", "ports": [9095], "running": 1, "size": 1}}] 2024-05-03T08:56:19.283 INFO:tasks.cephadm:mds.foofs has 0/2 2024-05-03T08:56:19.571 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:19 smithi123 ceph-mon[24496]: pgmap v220: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 280 B/s wr, 1 op/s 2024-05-03T08:56:19.571 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:19 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:19.571 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:19 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:56:19.572 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:19 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:19.572 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:19 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:56:19.572 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:19 smithi123 ceph-mon[24496]: from='client.14496 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-03T08:56:19.572 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:19 smithi123 ceph-mon[24496]: mds.? [v2:172.21.15.123:6832/1083595235,v1:172.21.15.123:6833/1083595235] up:boot 2024-05-03T08:56:19.572 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:19 smithi123 ceph-mon[24496]: fsmap foofs:1 {0=foofs.smithi032.axmmtz=up:active} 1 up:standby 2024-05-03T08:56:19.572 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:19 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.gtrlts"}]: dispatch 2024-05-03T08:56:19.572 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:19 smithi123 ceph-mon[24496]: fsmap foofs:1 {0=foofs.smithi032.axmmtz=up:active} 1 up:standby 2024-05-03T08:56:19.581 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:19 smithi032 ceph-mon[22403]: pgmap v220: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 280 B/s wr, 1 op/s 2024-05-03T08:56:19.581 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:19 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:19.581 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:19 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:56:19.581 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:19 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:19.581 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:19 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:56:19.581 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:19 smithi032 ceph-mon[22403]: from='client.14496 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-03T08:56:19.581 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:19 smithi032 ceph-mon[22403]: mds.? [v2:172.21.15.123:6832/1083595235,v1:172.21.15.123:6833/1083595235] up:boot 2024-05-03T08:56:19.582 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:19 smithi032 ceph-mon[22403]: fsmap foofs:1 {0=foofs.smithi032.axmmtz=up:active} 1 up:standby 2024-05-03T08:56:19.582 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:19 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.gtrlts"}]: dispatch 2024-05-03T08:56:19.582 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:19 smithi032 ceph-mon[22403]: fsmap foofs:1 {0=foofs.smithi032.axmmtz=up:active} 1 up:standby 2024-05-03T08:56:20.285 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph orch ls -f json 2024-05-03T08:56:21.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:21 smithi032 ceph-mon[22403]: pgmap v221: 65 pgs: 65 active+clean; 578 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 616 B/s wr, 4 op/s 2024-05-03T08:56:21.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:21 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:21.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:21 smithi123 ceph-mon[24496]: pgmap v221: 65 pgs: 65 active+clean; 578 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 616 B/s wr, 4 op/s 2024-05-03T08:56:21.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:21 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:23.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:22 smithi032 ceph-mon[22403]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-05-03T08:56:23.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:22 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:23.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:22 smithi123 ceph-mon[24496]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-05-03T08:56:23.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:22 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:23.246 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:56:23.246 INFO:teuthology.orchestra.run.smithi032.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-05-03T08:49:37.936309Z", "last_refresh": "2024-05-03T08:53:14.571967Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-05-03T08:49:31.820376Z", "last_refresh": "2024-05-03T08:53:14.572037Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-05-03T08:49:34.601976Z", "last_refresh": "2024-05-03T08:53:14.572102Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-05-03T08:56:10.489250Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-05-03T08:56:10.483051Z", "last_refresh": "2024-05-03T08:56:20.229466Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-05-03T08:49:29.676178Z", "last_refresh": "2024-05-03T08:53:14.571888Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi032:172.21.15.32=smithi032", "smithi123:172.21.15.123=smithi123"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-05-03T08:50:21.959405Z", "last_refresh": "2024-05-03T08:53:14.571732Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-05-03T08:49:36.655612Z", "last_refresh": "2024-05-03T08:53:14.572166Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-05-03T08:53:14.572295Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-05-03T08:49:33.325354Z", "last_refresh": "2024-05-03T08:53:14.572231Z", "ports": [9095], "running": 1, "size": 1}}] 2024-05-03T08:56:24.089 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:23 smithi032 ceph-mon[22403]: from='client.14500 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-03T08:56:24.113 INFO:tasks.cephadm:mds.foofs has 1/2 2024-05-03T08:56:24.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:23 smithi123 ceph-mon[24496]: from='client.14500 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-03T08:56:25.115 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- ceph orch ls -f json 2024-05-03T08:56:25.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:24 smithi032 ceph-mon[22403]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-05-03T08:56:25.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:24 smithi123 ceph-mon[24496]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-05-03T08:56:26.605 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:26 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:26.605 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:26 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:26.605 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:26 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:56:26.605 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:26 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:26.605 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:26 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:56:26.605 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:26 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:26.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:26 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:26.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:26 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:26.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:26 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:56:26.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:26 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:26.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:26 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:56:26.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:26 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:26.995 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:56:26.995 INFO:teuthology.orchestra.run.smithi032.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-05-03T08:49:37.936309Z", "last_refresh": "2024-05-03T08:56:25.397274Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-05-03T08:49:31.820376Z", "last_refresh": "2024-05-03T08:56:20.228854Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-05-03T08:49:34.601976Z", "last_refresh": "2024-05-03T08:56:25.397409Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-05-03T08:56:10.489250Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-05-03T08:56:10.483051Z", "last_refresh": "2024-05-03T08:56:20.229466Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-05-03T08:49:29.676178Z", "last_refresh": "2024-05-03T08:56:20.229009Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi032:172.21.15.32=smithi032", "smithi123:172.21.15.123=smithi123"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-05-03T08:50:21.959405Z", "last_refresh": "2024-05-03T08:56:20.229084Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-05-03T08:49:36.655612Z", "last_refresh": "2024-05-03T08:56:20.229150Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-05-03T08:56:20.229215Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-05-03T08:49:33.325354Z", "last_refresh": "2024-05-03T08:56:25.397538Z", "ports": [9095], "running": 1, "size": 1}}] 2024-05-03T08:56:27.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:27 smithi032 ceph-mon[22403]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-05-03T08:56:27.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:27 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:27.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:27 smithi123 ceph-mon[24496]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-05-03T08:56:27.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:27 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:27.772 INFO:tasks.cephadm:mds.foofs has 2/2 2024-05-03T08:56:27.773 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-03T08:56:27.782 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi032.front.sepia.ceph.com 2024-05-03T08:56:27.782 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-05-03T08:56:28.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:28 smithi123 ceph-mon[24496]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-03T08:56:28.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:28 smithi032 ceph-mon[22403]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-03T08:56:29.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:29 smithi123 ceph-mon[24496]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-05-03T08:56:29.890 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:29 smithi032 ceph-mon[22403]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-05-03T08:56:30.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:30 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-05-03T08:56:30.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:30 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-05-03T08:56:31.471 INFO:teuthology.orchestra.run.smithi032.stdout:NFS Cluster Created Successfully 2024-05-03T08:56:31.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:31 smithi123 ceph-mon[24496]: from='client.14508 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:31.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:31 smithi123 ceph-mon[24496]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 938 B/s wr, 2 op/s 2024-05-03T08:56:31.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:31 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-05-03T08:56:31.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:31 smithi123 ceph-mon[24496]: osdmap e65: 8 total, 8 up, 8 in 2024-05-03T08:56:31.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:31 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-05-03T08:56:31.742 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:31 smithi032 ceph-mon[22403]: from='client.14508 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:31.742 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:31 smithi032 ceph-mon[22403]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 938 B/s wr, 2 op/s 2024-05-03T08:56:31.742 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:31 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-05-03T08:56:31.742 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:31 smithi032 ceph-mon[22403]: osdmap e65: 8 total, 8 up, 8 in 2024-05-03T08:56:31.742 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:31 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-05-03T08:56:32.192 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-05-03T08:56:32.715 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-05-03T08:56:32.715 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:32 smithi032 ceph-mon[22403]: osdmap e66: 8 total, 8 up, 8 in 2024-05-03T08:56:32.715 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:32 smithi032 ceph-mon[22403]: Saving service nfs.foo spec with placement count:2 2024-05-03T08:56:32.715 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:32.715 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:56:32.715 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:32.715 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:56:32.715 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:32.715 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:32.715 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi032.myyxfz", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-05-03T08:56:32.715 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi032.myyxfz", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-05-03T08:56:32.715 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-05-03T08:56:32.715 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-05-03T08:56:32.716 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:32.716 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-05-03T08:56:32.716 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-05-03T08:56:32.716 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi032.myyxfz-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-03T08:56:32.716 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi032.myyxfz-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-03T08:56:32.716 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:32 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:32.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-05-03T08:56:32.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:32 smithi123 ceph-mon[24496]: osdmap e66: 8 total, 8 up, 8 in 2024-05-03T08:56:32.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:32 smithi123 ceph-mon[24496]: Saving service nfs.foo spec with placement count:2 2024-05-03T08:56:32.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:32.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:56:32.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:32.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:56:32.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:32.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:32.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi032.myyxfz", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-05-03T08:56:32.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi032.myyxfz", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-05-03T08:56:32.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-05-03T08:56:32.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-05-03T08:56:32.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:32.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-05-03T08:56:32.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-05-03T08:56:32.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi032.myyxfz-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-03T08:56:32.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi032.myyxfz-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-03T08:56:32.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:32 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:33.711 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:33 smithi032 ceph-mon[22403]: Creating key for client.nfs.foo.0.0.smithi032.myyxfz 2024-05-03T08:56:33.711 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:33 smithi032 ceph-mon[22403]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-05-03T08:56:33.711 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:33 smithi032 ceph-mon[22403]: Rados config object exists: conf-nfs.foo 2024-05-03T08:56:33.711 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:33 smithi032 ceph-mon[22403]: Creating key for client.nfs.foo.0.0.smithi032.myyxfz-rgw 2024-05-03T08:56:33.711 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:33 smithi032 ceph-mon[22403]: Deploying daemon nfs.foo.0.0.smithi032.myyxfz on smithi032 2024-05-03T08:56:33.711 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:33 smithi032 ceph-mon[22403]: pgmap v229: 97 pgs: 5 creating+peering, 22 unknown, 70 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:33.711 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:33 smithi032 ceph-mon[22403]: osdmap e67: 8 total, 8 up, 8 in 2024-05-03T08:56:33.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:33 smithi123 ceph-mon[24496]: Creating key for client.nfs.foo.0.0.smithi032.myyxfz 2024-05-03T08:56:33.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:33 smithi123 ceph-mon[24496]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-05-03T08:56:33.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:33 smithi123 ceph-mon[24496]: Rados config object exists: conf-nfs.foo 2024-05-03T08:56:33.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:33 smithi123 ceph-mon[24496]: Creating key for client.nfs.foo.0.0.smithi032.myyxfz-rgw 2024-05-03T08:56:33.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:33 smithi123 ceph-mon[24496]: Deploying daemon nfs.foo.0.0.smithi032.myyxfz on smithi032 2024-05-03T08:56:33.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:33 smithi123 ceph-mon[24496]: pgmap v229: 97 pgs: 5 creating+peering, 22 unknown, 70 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:33.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:33 smithi123 ceph-mon[24496]: osdmap e67: 8 total, 8 up, 8 in 2024-05-03T08:56:34.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:34 smithi032 ceph-mon[22403]: mgrmap e20: smithi032.umllay(active, since 5m), standbys: smithi123.xrxinh 2024-05-03T08:56:34.888 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:34 smithi032 ceph-mon[22403]: pgmap v231: 97 pgs: 5 creating+peering, 18 unknown, 74 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:34.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:34 smithi123 ceph-mon[24496]: mgrmap e20: smithi032.umllay(active, since 5m), standbys: smithi123.xrxinh 2024-05-03T08:56:34.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:34 smithi123 ceph-mon[24496]: pgmap v231: 97 pgs: 5 creating+peering, 18 unknown, 74 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-05-03T08:56:35.707 INFO:teuthology.orchestra.run.smithi032.stderr:Invalid command: Unexpected argument '--clusterid' 2024-05-03T08:56:35.707 INFO:teuthology.orchestra.run.smithi032.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-05-03T08:56:35.707 INFO:teuthology.orchestra.run.smithi032.stderr:Error EINVAL: invalid command 2024-05-03T08:56:36.065 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-05-03T08:56:36.065 INFO:teuthology.orchestra.run.smithi032.stdout: "bind": "/fake", 2024-05-03T08:56:36.065 INFO:teuthology.orchestra.run.smithi032.stdout: "fs": "foofs", 2024-05-03T08:56:36.065 INFO:teuthology.orchestra.run.smithi032.stdout: "path": "/", 2024-05-03T08:56:36.065 INFO:teuthology.orchestra.run.smithi032.stdout: "cluster": "foo", 2024-05-03T08:56:36.065 INFO:teuthology.orchestra.run.smithi032.stdout: "mode": "RW" 2024-05-03T08:56:36.065 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-05-03T08:56:36.862 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-05-03T08:56:37.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:36 smithi032 ceph-mon[22403]: pgmap v232: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-05-03T08:56:37.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:36 smithi032 ceph-mon[22403]: from='client.14530 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:37.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:36 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-05-03T08:56:37.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:36 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-05-03T08:56:37.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:36 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:37.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:36 smithi032 ceph-mon[22403]: Creating key for client.nfs.foo.1.0.smithi123.dkofzl 2024-05-03T08:56:37.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:36 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi123.dkofzl", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-05-03T08:56:37.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:36 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi123.dkofzl", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-05-03T08:56:37.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:36 smithi032 ceph-mon[22403]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-05-03T08:56:37.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:36 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-05-03T08:56:37.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:36 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-05-03T08:56:37.139 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:36 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:37.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:36 smithi123 ceph-mon[24496]: pgmap v232: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-05-03T08:56:37.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:36 smithi123 ceph-mon[24496]: from='client.14530 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:37.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:36 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-05-03T08:56:37.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:36 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-05-03T08:56:37.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:36 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:37.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:36 smithi123 ceph-mon[24496]: Creating key for client.nfs.foo.1.0.smithi123.dkofzl 2024-05-03T08:56:37.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:36 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi123.dkofzl", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-05-03T08:56:37.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:36 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi123.dkofzl", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-05-03T08:56:37.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:36 smithi123 ceph-mon[24496]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-05-03T08:56:37.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:36 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-05-03T08:56:37.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:36 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-05-03T08:56:37.228 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:36 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:39.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:38 smithi032 ceph-mon[22403]: pgmap v233: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 139 B/s rd, 278 B/s wr, 0 op/s 2024-05-03T08:56:39.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:38 smithi123 ceph-mon[24496]: pgmap v233: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 139 B/s rd, 278 B/s wr, 0 op/s 2024-05-03T08:56:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:39 smithi123 ceph-mon[24496]: from='client.14546 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:39 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-05-03T08:56:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:39 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-05-03T08:56:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:39 smithi123 ceph-mon[24496]: Rados config object exists: conf-nfs.foo 2024-05-03T08:56:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:39 smithi123 ceph-mon[24496]: Creating key for client.nfs.foo.1.0.smithi123.dkofzl-rgw 2024-05-03T08:56:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:39 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi123.dkofzl-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-03T08:56:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:39 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi123.dkofzl-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-03T08:56:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:39 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:39 smithi123 ceph-mon[24496]: Deploying daemon nfs.foo.1.0.smithi123.dkofzl on smithi123 2024-05-03T08:56:40.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:39 smithi032 ceph-mon[22403]: from='client.14546 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:40.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:39 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-05-03T08:56:40.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:39 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-05-03T08:56:40.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:39 smithi032 ceph-mon[22403]: Rados config object exists: conf-nfs.foo 2024-05-03T08:56:40.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:39 smithi032 ceph-mon[22403]: Creating key for client.nfs.foo.1.0.smithi123.dkofzl-rgw 2024-05-03T08:56:40.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:39 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi123.dkofzl-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-03T08:56:40.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:39 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi123.dkofzl-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-03T08:56:40.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:39 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:40.138 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:39 smithi032 ceph-mon[22403]: Deploying daemon nfs.foo.1.0.smithi123.dkofzl on smithi123 2024-05-03T08:56:41.289 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:41 smithi123 ceph-mon[24496]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-05-03T08:56:41.289 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:41 smithi123 ceph-mon[24496]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:41.382 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:41 smithi032 ceph-mon[22403]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-05-03T08:56:41.382 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:41 smithi032 ceph-mon[22403]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:56:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:42 smithi032 ceph-mon[22403]: from='client.14564 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:56:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]: dispatch 2024-05-03T08:56:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-05-03T08:56:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-05-03T08:56:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-05-03T08:56:42.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:56:42.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:56:42.639 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:42.719 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:42.720 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:56:42.720 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:42.720 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:42 smithi123 ceph-mon[24496]: from='client.14564 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:42.720 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:56:42.720 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]: dispatch 2024-05-03T08:56:42.720 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-05-03T08:56:42.720 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-05-03T08:56:42.720 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-05-03T08:56:42.720 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:56:42.720 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:56:42.720 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:43.630 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:43 smithi123 ceph-mon[24496]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-05-03T08:56:43.631 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:43 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]': finished 2024-05-03T08:56:43.631 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:43 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-05-03T08:56:43.631 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:43 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-05-03T08:56:43.631 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:43 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-05-03T08:56:43.631 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:43 smithi123 ceph-mon[24496]: osdmap e68: 8 total, 8 up, 8 in 2024-05-03T08:56:43.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:43 smithi032 ceph-mon[22403]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-05-03T08:56:43.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:43 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]': finished 2024-05-03T08:56:43.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:43 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-05-03T08:56:43.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:43 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-05-03T08:56:43.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:43 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-05-03T08:56:43.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:43 smithi032 ceph-mon[22403]: osdmap e68: 8 total, 8 up, 8 in 2024-05-03T08:56:44.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:44 smithi032 ceph-mon[22403]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:44.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:44 smithi032 ceph-mon[22403]: osdmap e69: 8 total, 8 up, 8 in 2024-05-03T08:56:44.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:44 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:44.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:44 smithi123 ceph-mon[24496]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:44.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:44 smithi123 ceph-mon[24496]: osdmap e69: 8 total, 8 up, 8 in 2024-05-03T08:56:44.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:44 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:45.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:45 smithi032 ceph-mon[22403]: pgmap v238: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 2.1 KiB/s wr, 4 op/s 2024-05-03T08:56:45.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:45 smithi032 ceph-mon[22403]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:45.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:45 smithi123 ceph-mon[24496]: pgmap v238: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 2.1 KiB/s wr, 4 op/s 2024-05-03T08:56:45.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:45 smithi123 ceph-mon[24496]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:46.634 INFO:teuthology.orchestra.run.smithi032.stdout:nfs.foo ?:2049 2/2 2s ago 15s count:2 2024-05-03T08:56:46.634 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:46 smithi032 ceph-mon[22403]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:46.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:46 smithi123 ceph-mon[24496]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:47.281 INFO:teuthology.run_tasks:Running task vip.exec... 2024-05-03T08:56:47.290 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi032.front.sepia.ceph.com 2024-05-03T08:56:47.290 DEBUG:teuthology.orchestra.run.smithi032:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-05-03T08:56:47.318 INFO:teuthology.orchestra.run.smithi032.stderr:+ mkdir /mnt/foo 2024-05-03T08:56:47.325 DEBUG:teuthology.orchestra.run.smithi032:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-05-03T08:56:47.393 INFO:teuthology.orchestra.run.smithi032.stderr:++ hostname 2024-05-03T08:56:47.393 INFO:teuthology.orchestra.run.smithi032.stderr:+ mount -t nfs smithi032:/fake /mnt/foo -o sync 2024-05-03T08:56:47.514 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:47 smithi032 ceph-mon[22403]: pgmap v239: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s 2024-05-03T08:56:47.514 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:47 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:47.514 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:47 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:47.514 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:47 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:47.514 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:47 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:56:47.514 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:47 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:47.514 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:47 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:56:47.514 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:47 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:47.514 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:47 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:47.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:47 smithi123 ceph-mon[24496]: pgmap v239: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s 2024-05-03T08:56:47.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:47 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:47.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:47 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:47.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:47 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:47.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:47 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:56:47.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:47 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:56:47.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:47 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:56:47.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:47 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:47.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:47 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:56:47.780 DEBUG:teuthology.orchestra.run.smithi032:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-05-03T08:56:47.844 INFO:teuthology.orchestra.run.smithi032.stderr:+ echo test 2024-05-03T08:56:47.886 DEBUG:teuthology.orchestra.run.smithi032:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-05-03T08:56:47.950 INFO:teuthology.orchestra.run.smithi032.stderr:+ sync 2024-05-03T08:56:48.111 INFO:teuthology.run_tasks:Running task parallel... 2024-05-03T08:56:48.136 INFO:teuthology.task.parallel:starting parallel... 2024-05-03T08:56:48.137 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-05-03T08:56:48.137 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-05-03T08:56:48.138 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi032.front.sepia.ceph.com 2024-05-03T08:56:48.138 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -e sha1=f8e6d2d721bc96a93915917c59bc3d287d57263d -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-05-03T08:56:48.138 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-05-03T08:56:48.138 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-05-03T08:56:48.139 INFO:teuthology.task.exec:Executing custom commands... 2024-05-03T08:56:48.139 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi032.front.sepia.ceph.com 2024-05-03T08:56:48.139 DEBUG:teuthology.orchestra.run.smithi032:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-05-03T08:56:48.229 INFO:teuthology.orchestra.run.smithi032.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-05-03T08:56:48.229 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-05-03T08:56:48.229 INFO:teuthology.orchestra.run.smithi032.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-05-03T08:56:48.230 INFO:teuthology.orchestra.run.smithi032.stdout:4 of 5 processes prepared for launch 0 sec 2024-05-03T08:56:48.330 INFO:teuthology.orchestra.run.smithi032.stdout:5 of 5 processes prepared for launch 0 sec 2024-05-03T08:56:48.331 INFO:teuthology.orchestra.run.smithi032.stdout:releasing clients 2024-05-03T08:56:48.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:48 smithi032 ceph-mon[22403]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:48.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:48 smithi123 ceph-mon[24496]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:56:49.331 INFO:teuthology.orchestra.run.smithi032.stdout: 5 136 29.79 MB/sec warmup 1 sec latency 127.911 ms 2024-05-03T08:56:49.410 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:49 smithi032 ceph-mon[22403]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.0 KiB/s wr, 5 op/s 2024-05-03T08:56:49.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:49 smithi123 ceph-mon[24496]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.0 KiB/s wr, 5 op/s 2024-05-03T08:56:50.331 INFO:teuthology.orchestra.run.smithi032.stdout: 5 368 32.81 MB/sec warmup 2 sec latency 29.113 ms 2024-05-03T08:56:51.331 INFO:teuthology.orchestra.run.smithi032.stdout: 5 571 32.60 MB/sec warmup 3 sec latency 26.000 ms 2024-05-03T08:56:51.440 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:51 smithi032 ceph-mon[22403]: pgmap v241: 97 pgs: 97 active+clean; 16 MiB data, 61 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.9 MiB/s wr, 39 op/s 2024-05-03T08:56:51.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:51 smithi123 ceph-mon[24496]: pgmap v241: 97 pgs: 97 active+clean; 16 MiB data, 61 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.9 MiB/s wr, 39 op/s 2024-05-03T08:56:52.331 INFO:teuthology.orchestra.run.smithi032.stdout: 5 1114 28.70 MB/sec warmup 4 sec latency 30.771 ms 2024-05-03T08:56:52.546 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -e sha1=f8e6d2d721bc96a93915917c59bc3d287d57263d -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-05-03T08:56:53.331 INFO:teuthology.orchestra.run.smithi032.stdout: 5 2086 26.65 MB/sec warmup 5 sec latency 16.250 ms 2024-05-03T08:56:53.452 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:53 smithi032 ceph-mon[22403]: pgmap v242: 97 pgs: 97 active+clean; 40 MiB data, 66 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.3 MiB/s wr, 84 op/s 2024-05-03T08:56:53.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:53 smithi123 ceph-mon[24496]: pgmap v242: 97 pgs: 97 active+clean; 40 MiB data, 66 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.3 MiB/s wr, 84 op/s 2024-05-03T08:56:54.331 INFO:teuthology.orchestra.run.smithi032.stdout: 5 2997 28.02 MB/sec warmup 6 sec latency 16.813 ms 2024-05-03T08:56:55.331 INFO:teuthology.orchestra.run.smithi032.stdout: 5 3752 27.98 MB/sec warmup 7 sec latency 22.062 ms 2024-05-03T08:56:55.473 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:55 smithi032 ceph-mon[22403]: pgmap v243: 97 pgs: 97 active+clean; 71 MiB data, 125 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 7.1 MiB/s wr, 285 op/s 2024-05-03T08:56:55.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:55 smithi123 ceph-mon[24496]: pgmap v243: 97 pgs: 97 active+clean; 71 MiB data, 125 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 7.1 MiB/s wr, 285 op/s 2024-05-03T08:56:56.336 INFO:teuthology.orchestra.run.smithi032.stdout: 5 4390 25.37 MB/sec warmup 8 sec latency 22.626 ms 2024-05-03T08:56:57.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 5321 25.21 MB/sec warmup 9 sec latency 16.679 ms 2024-05-03T08:56:57.482 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:57 smithi032 ceph-mon[22403]: pgmap v244: 97 pgs: 97 active+clean; 123 MiB data, 204 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 12 MiB/s wr, 566 op/s 2024-05-03T08:56:57.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:57 smithi123 ceph-mon[24496]: pgmap v244: 97 pgs: 97 active+clean; 123 MiB data, 204 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 12 MiB/s wr, 566 op/s 2024-05-03T08:56:58.110 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -e sha1=f8e6d2d721bc96a93915917c59bc3d287d57263d -- bash -c 'ceph config set global log_to_journald false --force' 2024-05-03T08:56:58.336 INFO:teuthology.orchestra.run.smithi032.stdout: 5 6160 24.46 MB/sec warmup 10 sec latency 18.990 ms 2024-05-03T08:56:59.336 INFO:teuthology.orchestra.run.smithi032.stdout: 5 7074 25.49 MB/sec warmup 11 sec latency 22.298 ms 2024-05-03T08:56:59.501 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:56:59 smithi032 ceph-mon[22403]: pgmap v245: 97 pgs: 97 active+clean; 130 MiB data, 237 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 13 MiB/s wr, 708 op/s 2024-05-03T08:56:59.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:56:59 smithi123 ceph-mon[24496]: pgmap v245: 97 pgs: 97 active+clean; 130 MiB data, 237 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 13 MiB/s wr, 708 op/s 2024-05-03T08:57:00.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 7832 24.79 MB/sec warmup 12 sec latency 18.804 ms 2024-05-03T08:57:01.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 8742 24.76 MB/sec warmup 13 sec latency 10.940 ms 2024-05-03T08:57:01.511 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:01 smithi032 ceph-mon[22403]: pgmap v246: 97 pgs: 97 active+clean; 140 MiB data, 283 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 16 MiB/s wr, 978 op/s 2024-05-03T08:57:01.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:01 smithi123 ceph-mon[24496]: pgmap v246: 97 pgs: 97 active+clean; 140 MiB data, 283 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 16 MiB/s wr, 978 op/s 2024-05-03T08:57:02.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 9567 23.96 MB/sec warmup 14 sec latency 24.498 ms 2024-05-03T08:57:03.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 10587 25.04 MB/sec warmup 15 sec latency 22.729 ms 2024-05-03T08:57:03.400 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -e sha1=f8e6d2d721bc96a93915917c59bc3d287d57263d -- bash -c 'ceph mgr module enable nfs --force' 2024-05-03T08:57:03.578 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:03 smithi032 ceph-mon[22403]: pgmap v247: 97 pgs: 97 active+clean; 141 MiB data, 307 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 15 MiB/s wr, 1.02k op/s 2024-05-03T08:57:03.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:03 smithi123 ceph-mon[24496]: pgmap v247: 97 pgs: 97 active+clean; 141 MiB data, 307 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 15 MiB/s wr, 1.02k op/s 2024-05-03T08:57:04.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 11166 24.49 MB/sec warmup 16 sec latency 35.023 ms 2024-05-03T08:57:05.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 11929 23.98 MB/sec warmup 17 sec latency 13.928 ms 2024-05-03T08:57:05.597 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:05 smithi032 ceph-mon[22403]: pgmap v248: 97 pgs: 97 active+clean; 148 MiB data, 345 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 15 MiB/s wr, 1.19k op/s 2024-05-03T08:57:05.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:05 smithi123 ceph-mon[24496]: pgmap v248: 97 pgs: 97 active+clean; 148 MiB data, 345 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 15 MiB/s wr, 1.19k op/s 2024-05-03T08:57:06.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 12960 23.54 MB/sec warmup 18 sec latency 14.202 ms 2024-05-03T08:57:07.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 13782 24.22 MB/sec warmup 19 sec latency 28.856 ms 2024-05-03T08:57:07.441 INFO:teuthology.orchestra.run.smithi032.stderr:module 'nfs' is already enabled 2024-05-03T08:57:07.605 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:07 smithi032 ceph-mon[22403]: pgmap v249: 97 pgs: 97 active+clean; 153 MiB data, 413 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 15 MiB/s wr, 1.20k op/s 2024-05-03T08:57:07.606 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:07 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/2794490404' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-05-03T08:57:07.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:07 smithi123 ceph-mon[24496]: pgmap v249: 97 pgs: 97 active+clean; 153 MiB data, 413 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 15 MiB/s wr, 1.20k op/s 2024-05-03T08:57:07.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:07 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/2794490404' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-05-03T08:57:08.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 14533 24.31 MB/sec warmup 20 sec latency 16.439 ms 2024-05-03T08:57:08.529 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -e sha1=f8e6d2d721bc96a93915917c59bc3d287d57263d -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-05-03T08:57:08.609 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:08 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/2794490404' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-05-03T08:57:08.610 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:08 smithi032 ceph-mon[22403]: mgrmap e21: smithi032.umllay(active, since 6m), standbys: smithi123.xrxinh 2024-05-03T08:57:08.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:08 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/2794490404' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-05-03T08:57:08.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:08 smithi123 ceph-mon[24496]: mgrmap e21: smithi032.umllay(active, since 6m), standbys: smithi123.xrxinh 2024-05-03T08:57:09.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 15121 23.41 MB/sec warmup 21 sec latency 29.549 ms 2024-05-03T08:57:09.616 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:09 smithi032 ceph-mon[22403]: pgmap v250: 97 pgs: 97 active+clean; 159 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-05-03T08:57:09.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:09 smithi123 ceph-mon[24496]: pgmap v250: 97 pgs: 97 active+clean; 159 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-05-03T08:57:10.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 16028 23.41 MB/sec warmup 22 sec latency 17.976 ms 2024-05-03T08:57:11.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 17083 23.57 MB/sec warmup 23 sec latency 18.798 ms 2024-05-03T08:57:11.626 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:11 smithi032 ceph-mon[22403]: pgmap v251: 97 pgs: 97 active+clean; 163 MiB data, 481 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 12 MiB/s wr, 1.08k op/s 2024-05-03T08:57:11.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:11 smithi123 ceph-mon[24496]: pgmap v251: 97 pgs: 97 active+clean; 163 MiB data, 481 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 12 MiB/s wr, 1.08k op/s 2024-05-03T08:57:12.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 18026 24.15 MB/sec warmup 24 sec latency 19.583 ms 2024-05-03T08:57:12.960 INFO:teuthology.orchestra.run.smithi032.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:f8e6d2d721bc96a93915917c59bc3d287d57263d 2024-05-03T08:57:13.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 18553 23.43 MB/sec warmup 25 sec latency 18.790 ms 2024-05-03T08:57:13.636 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:13 smithi032 ceph-mon[22403]: pgmap v252: 97 pgs: 97 active+clean; 166 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 934 op/s 2024-05-03T08:57:13.637 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:13 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:57:13.637 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:13 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:57:13.637 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:13 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:57:13.637 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:13 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:57:13.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:13 smithi123 ceph-mon[24496]: pgmap v252: 97 pgs: 97 active+clean; 166 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 934 op/s 2024-05-03T08:57:13.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:13 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:57:13.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:13 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:57:13.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:13 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:57:13.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:13 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:57:14.316 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-05-03T08:57:14.317 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi032.front.sepia.ceph.com 2024-05-03T08:57:14.317 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -e sha1=f8e6d2d721bc96a93915917c59bc3d287d57263d -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-05-03T08:57:14.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 19325 23.20 MB/sec warmup 26 sec latency 18.044 ms 2024-05-03T08:57:14.633 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:14 smithi032 ceph-mon[22403]: from='client.24409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:f8e6d2d721bc96a93915917c59bc3d287d57263d", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:57:14.633 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:14 smithi032 ceph-mon[22403]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:f8e6d2d721bc96a93915917c59bc3d287d57263d 2024-05-03T08:57:14.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:14 smithi123 ceph-mon[24496]: from='client.24409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:f8e6d2d721bc96a93915917c59bc3d287d57263d", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:57:14.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:14 smithi123 ceph-mon[24496]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:f8e6d2d721bc96a93915917c59bc3d287d57263d 2024-05-03T08:57:15.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 20026 22.81 MB/sec warmup 27 sec latency 31.237 ms 2024-05-03T08:57:15.650 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:15 smithi032 ceph-mon[22403]: pgmap v253: 97 pgs: 97 active+clean; 171 MiB data, 562 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-05-03T08:57:15.651 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:15 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:57:15.651 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:15 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:57:15.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:15 smithi123 ceph-mon[24496]: pgmap v253: 97 pgs: 97 active+clean; 171 MiB data, 562 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-05-03T08:57:15.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:15 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:57:15.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:15 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:57:16.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 20611 22.70 MB/sec warmup 28 sec latency 37.470 ms 2024-05-03T08:57:16.656 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:16 smithi032 ceph-mon[22403]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:f8e6d2d721bc96a93915917c59bc3d287d57263d 2024-05-03T08:57:16.657 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:16 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:57:16.657 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:16 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:57:16.657 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:16 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:57:16.657 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:16 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:57:16.657 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:16 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:57:16.657 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:16 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:57:16.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:16 smithi123 ceph-mon[24496]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:f8e6d2d721bc96a93915917c59bc3d287d57263d 2024-05-03T08:57:16.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:16 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:57:16.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:16 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:57:16.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:16 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:57:16.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:16 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:57:16.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:16 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:57:16.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:16 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:57:17.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 21192 22.70 MB/sec warmup 29 sec latency 31.302 ms 2024-05-03T08:57:17.637 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:17 smithi032 ceph-mon[22403]: Upgrade: Paused due to UPGRADE_FAILED_PULL: Upgrade: failed to pull target image 2024-05-03T08:57:17.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:17 smithi032 ceph-mon[22403]: pgmap v254: 97 pgs: 97 active+clean; 174 MiB data, 621 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 988 op/s 2024-05-03T08:57:17.638 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:17 smithi032 ceph-mon[22403]: Health check failed: Upgrade: failed to pull target image (UPGRADE_FAILED_PULL) 2024-05-03T08:57:17.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:17 smithi123 ceph-mon[24496]: Upgrade: Paused due to UPGRADE_FAILED_PULL: Upgrade: failed to pull target image 2024-05-03T08:57:17.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:17 smithi123 ceph-mon[24496]: pgmap v254: 97 pgs: 97 active+clean; 174 MiB data, 621 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 988 op/s 2024-05-03T08:57:17.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:17 smithi123 ceph-mon[24496]: Health check failed: Upgrade: failed to pull target image (UPGRADE_FAILED_PULL) 2024-05-03T08:57:17.799 INFO:teuthology.orchestra.run.smithi032.stdout:true 2024-05-03T08:57:18.280 INFO:teuthology.orchestra.run.smithi032.stdout:"Error: UPGRADE_FAILED_PULL: Upgrade: failed to pull target image" 2024-05-03T08:57:18.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 21843 22.63 MB/sec warmup 30 sec latency 24.584 ms 2024-05-03T08:57:19.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 22466 22.23 MB/sec warmup 31 sec latency 32.562 ms 2024-05-03T08:57:19.482 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -e sha1=f8e6d2d721bc96a93915917c59bc3d287d57263d -- bash -c 'ceph orch ps' 2024-05-03T08:57:19.650 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:19 smithi032 ceph-mon[22403]: pgmap v255: 97 pgs: 97 active+clean; 180 MiB data, 638 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 981 op/s 2024-05-03T08:57:19.650 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:19 smithi032 ceph-mon[22403]: from='client.14614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:57:19.650 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:19 smithi032 ceph-mon[22403]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:57:19.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:19 smithi123 ceph-mon[24496]: pgmap v255: 97 pgs: 97 active+clean; 180 MiB data, 638 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 981 op/s 2024-05-03T08:57:19.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:19 smithi123 ceph-mon[24496]: from='client.14614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:57:19.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:19 smithi123 ceph-mon[24496]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:57:20.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 23147 22.08 MB/sec warmup 32 sec latency 21.658 ms 2024-05-03T08:57:21.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 23891 21.82 MB/sec warmup 33 sec latency 40.371 ms 2024-05-03T08:57:21.678 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:21 smithi032 ceph-mon[22403]: pgmap v256: 97 pgs: 97 active+clean; 189 MiB data, 646 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-05-03T08:57:21.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:21 smithi123 ceph-mon[24496]: pgmap v256: 97 pgs: 97 active+clean; 189 MiB data, 646 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-05-03T08:57:22.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 24569 22.04 MB/sec warmup 34 sec latency 24.911 ms 2024-05-03T08:57:23.105 INFO:teuthology.orchestra.run.smithi032.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-03T08:57:23.105 INFO:teuthology.orchestra.run.smithi032.stdout:alertmanager.smithi032 smithi032 *:9093,9094 running (5m) 36s ago 7m 24.3M - ba2b418f427c ddf53d5d27fc 2024-05-03T08:57:23.105 INFO:teuthology.orchestra.run.smithi032.stdout:crash.smithi032 smithi032 running (7m) 36s ago 7m 6409k - 17.2.0 e1d6a67b021e 38dc8227bd42 2024-05-03T08:57:23.105 INFO:teuthology.orchestra.run.smithi032.stdout:crash.smithi123 smithi123 running (6m) 39s ago 6m 6405k - 17.2.0 e1d6a67b021e 2bce696b8db8 2024-05-03T08:57:23.105 INFO:teuthology.orchestra.run.smithi032.stdout:grafana.smithi032 smithi032 *:3000 running (5m) 36s ago 6m 54.4M - 8.3.5 dad864ee21e9 81ca1cc63b54 2024-05-03T08:57:23.105 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foofs.smithi032.axmmtz smithi032 running (68s) 36s ago 67s 15.8M - 17.2.0 e1d6a67b021e 0c91c3379b95 2024-05-03T08:57:23.106 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foofs.smithi123.gtrlts smithi123 running (65s) 39s ago 64s 13.4M - 17.2.0 e1d6a67b021e b24c9351f251 2024-05-03T08:57:23.106 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.smithi032.umllay smithi032 *:9283 running (8m) 36s ago 8m 460M - 17.2.0 e1d6a67b021e 537e53955aba 2024-05-03T08:57:23.106 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.smithi123.xrxinh smithi123 *:8443,9283 running (6m) 39s ago 6m 412M - 17.2.0 e1d6a67b021e e7499f622d47 2024-05-03T08:57:23.106 INFO:teuthology.orchestra.run.smithi032.stdout:mon.smithi032 smithi032 running (8m) 36s ago 8m 48.2M 2048M 17.2.0 e1d6a67b021e 7e5da84a501e 2024-05-03T08:57:23.106 INFO:teuthology.orchestra.run.smithi032.stdout:mon.smithi123 smithi123 running (6m) 39s ago 6m 39.0M 2048M 17.2.0 e1d6a67b021e 3a32332bb758 2024-05-03T08:57:23.106 INFO:teuthology.orchestra.run.smithi032.stdout:nfs.foo.0.0.smithi032.myyxfz smithi032 *:2049 running (47s) 36s ago 47s 53.1M - 4.0 e1d6a67b021e adc1bba0e7ed 2024-05-03T08:57:23.106 INFO:teuthology.orchestra.run.smithi032.stdout:nfs.foo.1.0.smithi123.dkofzl smithi123 *:2049 running (42s) 39s ago 41s 52.9M - 4.0 e1d6a67b021e f5ab76e466b3 2024-05-03T08:57:23.106 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.smithi032 smithi032 *:9100 running (6m) 36s ago 6m 17.0M - 1dbe0e931976 ae21c41038e9 2024-05-03T08:57:23.106 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.smithi123 smithi123 *:9100 running (6m) 39s ago 6m 15.9M - 1dbe0e931976 dabbb678f527 2024-05-03T08:57:23.106 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 smithi032 running (5m) 36s ago 5m 43.7M 3217M 17.2.0 e1d6a67b021e 167471b12229 2024-05-03T08:57:23.106 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 smithi032 running (5m) 36s ago 5m 46.2M 3217M 17.2.0 e1d6a67b021e 21b9c0b15d6f 2024-05-03T08:57:23.106 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 smithi032 running (4m) 36s ago 4m 39.0M 3217M 17.2.0 e1d6a67b021e 6dfd926898e3 2024-05-03T08:57:23.106 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 smithi032 running (4m) 36s ago 4m 40.6M 3217M 17.2.0 e1d6a67b021e 947099fbe826 2024-05-03T08:57:23.106 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 smithi123 running (3m) 39s ago 3m 40.6M 3984M 17.2.0 e1d6a67b021e 150ddc5685f9 2024-05-03T08:57:23.107 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 smithi123 running (3m) 39s ago 3m 42.0M 3984M 17.2.0 e1d6a67b021e 72ba92ce4b53 2024-05-03T08:57:23.107 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 smithi123 running (3m) 39s ago 3m 42.0M 3984M 17.2.0 e1d6a67b021e 2bc752f6bfc5 2024-05-03T08:57:23.107 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 smithi123 running (2m) 39s ago 2m 41.7M 3984M 17.2.0 e1d6a67b021e 40d54616bfbe 2024-05-03T08:57:23.107 INFO:teuthology.orchestra.run.smithi032.stdout:prometheus.smithi032 smithi032 *:9095 running (6m) 36s ago 6m 62.5M - 514e6a882f6e 6763db48c356 2024-05-03T08:57:23.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 25106 22.02 MB/sec warmup 35 sec latency 25.338 ms 2024-05-03T08:57:23.724 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:23 smithi032 ceph-mon[22403]: pgmap v257: 97 pgs: 97 active+clean; 188 MiB data, 663 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 9.8 MiB/s wr, 900 op/s 2024-05-03T08:57:23.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:23 smithi123 ceph-mon[24496]: pgmap v257: 97 pgs: 97 active+clean; 188 MiB data, 663 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 9.8 MiB/s wr, 900 op/s 2024-05-03T08:57:24.220 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -e sha1=f8e6d2d721bc96a93915917c59bc3d287d57263d -- bash -c 'ceph versions' 2024-05-03T08:57:24.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 25424 21.54 MB/sec warmup 36 sec latency 35.455 ms 2024-05-03T08:57:24.485 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:24 smithi032 ceph-mon[22403]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:57:24.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:24 smithi123 ceph-mon[24496]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:57:25.337 INFO:teuthology.orchestra.run.smithi032.stdout: 5 25941 21.15 MB/sec warmup 37 sec latency 55.098 ms 2024-05-03T08:57:25.492 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:25 smithi032 ceph-mon[22403]: pgmap v258: 97 pgs: 97 active+clean; 195 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 10 MiB/s wr, 991 op/s 2024-05-03T08:57:25.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:25 smithi123 ceph-mon[24496]: pgmap v258: 97 pgs: 97 active+clean; 195 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 10 MiB/s wr, 991 op/s 2024-05-03T08:57:26.338 INFO:teuthology.orchestra.run.smithi032.stdout: 5 26546 21.09 MB/sec warmup 38 sec latency 26.353 ms 2024-05-03T08:57:27.338 INFO:teuthology.orchestra.run.smithi032.stdout: 5 27354 20.85 MB/sec warmup 39 sec latency 22.521 ms 2024-05-03T08:57:27.516 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:27 smithi032 ceph-mon[22403]: pgmap v259: 97 pgs: 97 active+clean; 195 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.8 MiB/s wr, 913 op/s 2024-05-03T08:57:27.516 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:27 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:57:27.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:27 smithi123 ceph-mon[24496]: pgmap v259: 97 pgs: 97 active+clean; 195 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.8 MiB/s wr, 913 op/s 2024-05-03T08:57:27.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:27 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:57:28.338 INFO:teuthology.orchestra.run.smithi032.stdout: 5 27972 21.03 MB/sec warmup 40 sec latency 24.684 ms 2024-05-03T08:57:28.797 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-05-03T08:57:28.798 INFO:teuthology.orchestra.run.smithi032.stdout: "mon": { 2024-05-03T08:57:28.798 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-03T08:57:28.798 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-05-03T08:57:28.798 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr": { 2024-05-03T08:57:28.798 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-03T08:57:28.798 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-05-03T08:57:28.798 INFO:teuthology.orchestra.run.smithi032.stdout: "osd": { 2024-05-03T08:57:28.798 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-03T08:57:28.798 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-05-03T08:57:28.798 INFO:teuthology.orchestra.run.smithi032.stdout: "mds": { 2024-05-03T08:57:28.798 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-03T08:57:28.798 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-05-03T08:57:28.798 INFO:teuthology.orchestra.run.smithi032.stdout: "overall": { 2024-05-03T08:57:28.798 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-05-03T08:57:28.799 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-05-03T08:57:28.799 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-05-03T08:57:29.338 INFO:teuthology.orchestra.run.smithi032.stdout: 5 28637 21.10 MB/sec warmup 41 sec latency 27.458 ms 2024-05-03T08:57:29.545 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:29 smithi032 ceph-mon[22403]: pgmap v260: 97 pgs: 97 active+clean; 198 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 878 op/s 2024-05-03T08:57:29.545 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:29 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/2388592799' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-03T08:57:29.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:29 smithi123 ceph-mon[24496]: pgmap v260: 97 pgs: 97 active+clean; 198 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 878 op/s 2024-05-03T08:57:29.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:29 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/2388592799' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-03T08:57:30.049 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -e sha1=f8e6d2d721bc96a93915917c59bc3d287d57263d -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-05-03T08:57:30.338 INFO:teuthology.orchestra.run.smithi032.stdout: 5 29196 20.84 MB/sec warmup 42 sec latency 25.432 ms 2024-05-03T08:57:31.338 INFO:teuthology.orchestra.run.smithi032.stdout: 5 29927 20.72 MB/sec warmup 43 sec latency 61.483 ms 2024-05-03T08:57:31.553 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:31 smithi032 ceph-mon[22403]: pgmap v261: 97 pgs: 97 active+clean; 198 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.7 MiB/s wr, 718 op/s 2024-05-03T08:57:31.778 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:31 smithi123 ceph-mon[24496]: pgmap v261: 97 pgs: 97 active+clean; 198 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.7 MiB/s wr, 718 op/s 2024-05-03T08:57:32.338 INFO:teuthology.orchestra.run.smithi032.stdout: 5 30688 20.58 MB/sec warmup 44 sec latency 47.783 ms 2024-05-03T08:57:33.338 INFO:teuthology.orchestra.run.smithi032.stdout: 5 31222 20.48 MB/sec warmup 45 sec latency 30.811 ms 2024-05-03T08:57:33.496 INFO:teuthology.orchestra.run.smithi032.stdout:wait for servicemap items w/ changing names to refresh 2024-05-03T08:57:33.617 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:33 smithi032 ceph-mon[22403]: pgmap v262: 97 pgs: 97 active+clean; 203 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 687 op/s 2024-05-03T08:57:33.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:33 smithi123 ceph-mon[24496]: pgmap v262: 97 pgs: 97 active+clean; 203 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 687 op/s 2024-05-03T08:57:34.338 INFO:teuthology.orchestra.run.smithi032.stdout: 5 31852 20.56 MB/sec warmup 46 sec latency 39.088 ms 2024-05-03T08:57:34.633 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -e sha1=f8e6d2d721bc96a93915917c59bc3d287d57263d -- bash -c 'sleep 60' 2024-05-03T08:57:35.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 32386 20.53 MB/sec warmup 47 sec latency 23.830 ms 2024-05-03T08:57:35.526 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:35 smithi032 ceph-mon[22403]: pgmap v263: 97 pgs: 97 active+clean; 218 MiB data, 895 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.3 MiB/s wr, 943 op/s 2024-05-03T08:57:35.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:35 smithi123 ceph-mon[24496]: pgmap v263: 97 pgs: 97 active+clean; 218 MiB data, 895 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.3 MiB/s wr, 943 op/s 2024-05-03T08:57:36.338 INFO:teuthology.orchestra.run.smithi032.stdout: 5 32894 20.26 MB/sec warmup 48 sec latency 24.828 ms 2024-05-03T08:57:37.338 INFO:teuthology.orchestra.run.smithi032.stdout: 5 33526 20.19 MB/sec warmup 49 sec latency 23.517 ms 2024-05-03T08:57:37.554 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:37 smithi032 ceph-mon[22403]: pgmap v264: 97 pgs: 97 active+clean; 219 MiB data, 912 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 806 op/s 2024-05-03T08:57:37.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:37 smithi123 ceph-mon[24496]: pgmap v264: 97 pgs: 97 active+clean; 219 MiB data, 912 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 806 op/s 2024-05-03T08:57:38.338 INFO:teuthology.orchestra.run.smithi032.stdout: 5 34277 20.03 MB/sec warmup 50 sec latency 30.177 ms 2024-05-03T08:57:39.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 34965 20.14 MB/sec warmup 51 sec latency 24.994 ms 2024-05-03T08:57:39.630 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:39 smithi032 ceph-mon[22403]: pgmap v265: 97 pgs: 97 active+clean; 217 MiB data, 924 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 801 op/s 2024-05-03T08:57:39.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:39 smithi123 ceph-mon[24496]: pgmap v265: 97 pgs: 97 active+clean; 217 MiB data, 924 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 801 op/s 2024-05-03T08:57:40.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 35631 20.23 MB/sec warmup 52 sec latency 31.545 ms 2024-05-03T08:57:41.338 INFO:teuthology.orchestra.run.smithi032.stdout: 5 36075 20.05 MB/sec warmup 53 sec latency 27.680 ms 2024-05-03T08:57:41.641 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:41 smithi032 ceph-mon[22403]: pgmap v266: 97 pgs: 97 active+clean; 223 MiB data, 981 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.3 MiB/s wr, 835 op/s 2024-05-03T08:57:41.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:41 smithi123 ceph-mon[24496]: pgmap v266: 97 pgs: 97 active+clean; 223 MiB data, 981 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.3 MiB/s wr, 835 op/s 2024-05-03T08:57:42.338 INFO:teuthology.orchestra.run.smithi032.stdout: 5 36668 19.84 MB/sec warmup 54 sec latency 21.600 ms 2024-05-03T08:57:42.646 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:57:42.647 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:57:42.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:57:42.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:57:43.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 37175 19.74 MB/sec warmup 55 sec latency 57.706 ms 2024-05-03T08:57:43.653 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:43 smithi032 ceph-mon[22403]: pgmap v267: 97 pgs: 97 active+clean; 226 MiB data, 1015 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 938 op/s 2024-05-03T08:57:43.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:43 smithi123 ceph-mon[24496]: pgmap v267: 97 pgs: 97 active+clean; 226 MiB data, 1015 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 938 op/s 2024-05-03T08:57:44.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 37889 19.63 MB/sec warmup 56 sec latency 58.827 ms 2024-05-03T08:57:45.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 38720 19.83 MB/sec warmup 57 sec latency 18.996 ms 2024-05-03T08:57:45.670 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:45 smithi032 ceph-mon[22403]: pgmap v268: 97 pgs: 97 active+clean; 237 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.6 MiB/s wr, 1.02k op/s 2024-05-03T08:57:45.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:45 smithi123 ceph-mon[24496]: pgmap v268: 97 pgs: 97 active+clean; 237 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.6 MiB/s wr, 1.02k op/s 2024-05-03T08:57:46.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 39372 19.91 MB/sec warmup 58 sec latency 29.678 ms 2024-05-03T08:57:47.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 39846 19.66 MB/sec warmup 59 sec latency 34.813 ms 2024-05-03T08:57:47.692 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:47 smithi032 ceph-mon[22403]: pgmap v269: 97 pgs: 97 active+clean; 237 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.7 MiB/s wr, 762 op/s 2024-05-03T08:57:47.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:47 smithi123 ceph-mon[24496]: pgmap v269: 97 pgs: 97 active+clean; 237 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.7 MiB/s wr, 762 op/s 2024-05-03T08:57:48.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 40495 19.56 MB/sec warmup 60 sec latency 23.098 ms 2024-05-03T08:57:48.697 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:48 smithi032 ceph-mon[22403]: pgmap v270: 97 pgs: 97 active+clean; 237 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.4 KiB/s rd, 8.8 MiB/s wr, 774 op/s 2024-05-03T08:57:48.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:48 smithi123 ceph-mon[24496]: pgmap v270: 97 pgs: 97 active+clean; 237 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.4 KiB/s rd, 8.8 MiB/s wr, 774 op/s 2024-05-03T08:57:49.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 41268 19.52 MB/sec warmup 61 sec latency 29.009 ms 2024-05-03T08:57:50.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 41961 19.61 MB/sec warmup 62 sec latency 16.565 ms 2024-05-03T08:57:50.964 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:50 smithi032 ceph-mon[22403]: pgmap v271: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.6 MiB/s wr, 915 op/s 2024-05-03T08:57:51.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:50 smithi123 ceph-mon[24496]: pgmap v271: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.6 MiB/s wr, 915 op/s 2024-05-03T08:57:51.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 42663 19.72 MB/sec warmup 63 sec latency 25.701 ms 2024-05-03T08:57:52.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 43089 19.57 MB/sec warmup 64 sec latency 33.063 ms 2024-05-03T08:57:52.972 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:52 smithi032 ceph-mon[22403]: pgmap v272: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 804 op/s 2024-05-03T08:57:53.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:52 smithi123 ceph-mon[24496]: pgmap v272: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 804 op/s 2024-05-03T08:57:53.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 43672 19.40 MB/sec warmup 65 sec latency 32.071 ms 2024-05-03T08:57:54.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 44139 19.25 MB/sec warmup 66 sec latency 51.951 ms 2024-05-03T08:57:54.991 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:54 smithi032 ceph-mon[22403]: pgmap v273: 97 pgs: 97 active+clean; 261 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.4 MiB/s wr, 958 op/s 2024-05-03T08:57:55.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:54 smithi123 ceph-mon[24496]: pgmap v273: 97 pgs: 97 active+clean; 261 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.4 MiB/s wr, 958 op/s 2024-05-03T08:57:55.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 44658 19.14 MB/sec warmup 67 sec latency 88.109 ms 2024-05-03T08:57:56.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 45119 19.01 MB/sec warmup 68 sec latency 61.099 ms 2024-05-03T08:57:57.020 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:56 smithi032 ceph-mon[22403]: pgmap v274: 97 pgs: 97 active+clean; 258 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.8 MiB/s wr, 762 op/s 2024-05-03T08:57:57.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:56 smithi123 ceph-mon[24496]: pgmap v274: 97 pgs: 97 active+clean; 258 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.8 MiB/s wr, 762 op/s 2024-05-03T08:57:57.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 45654 19.06 MB/sec warmup 69 sec latency 61.160 ms 2024-05-03T08:57:58.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 46227 19.13 MB/sec warmup 70 sec latency 89.707 ms 2024-05-03T08:57:59.047 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:57:58 smithi032 ceph-mon[22403]: pgmap v275: 97 pgs: 97 active+clean; 257 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.8 MiB/s wr, 852 op/s 2024-05-03T08:57:59.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:57:58 smithi123 ceph-mon[24496]: pgmap v275: 97 pgs: 97 active+clean; 257 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.8 MiB/s wr, 852 op/s 2024-05-03T08:57:59.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 46688 19.00 MB/sec warmup 71 sec latency 41.323 ms 2024-05-03T08:58:00.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 47202 18.84 MB/sec warmup 72 sec latency 29.691 ms 2024-05-03T08:58:01.057 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:00 smithi032 ceph-mon[22403]: pgmap v276: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.0 MiB/s wr, 828 op/s 2024-05-03T08:58:01.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:00 smithi123 ceph-mon[24496]: pgmap v276: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.0 MiB/s wr, 828 op/s 2024-05-03T08:58:01.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 47943 18.85 MB/sec warmup 73 sec latency 32.374 ms 2024-05-03T08:58:02.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 48715 18.79 MB/sec warmup 74 sec latency 20.109 ms 2024-05-03T08:58:03.068 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:02 smithi032 ceph-mon[22403]: pgmap v277: 97 pgs: 97 active+clean; 268 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 8.9 MiB/s wr, 763 op/s 2024-05-03T08:58:03.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:02 smithi123 ceph-mon[24496]: pgmap v277: 97 pgs: 97 active+clean; 268 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 8.9 MiB/s wr, 763 op/s 2024-05-03T08:58:03.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 49332 18.91 MB/sec warmup 75 sec latency 35.046 ms 2024-05-03T08:58:04.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 49990 18.94 MB/sec warmup 76 sec latency 29.111 ms 2024-05-03T08:58:05.085 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:04 smithi032 ceph-mon[22403]: pgmap v278: 97 pgs: 97 active+clean; 270 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.2 MiB/s wr, 817 op/s 2024-05-03T08:58:05.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:04 smithi123 ceph-mon[24496]: pgmap v278: 97 pgs: 97 active+clean; 270 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.2 MiB/s wr, 817 op/s 2024-05-03T08:58:05.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 50571 18.83 MB/sec warmup 77 sec latency 25.233 ms 2024-05-03T08:58:06.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 51177 18.74 MB/sec warmup 78 sec latency 38.144 ms 2024-05-03T08:58:07.103 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:06 smithi032 ceph-mon[22403]: pgmap v279: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 659 op/s 2024-05-03T08:58:07.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:06 smithi123 ceph-mon[24496]: pgmap v279: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 659 op/s 2024-05-03T08:58:07.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 51905 18.71 MB/sec warmup 79 sec latency 94.084 ms 2024-05-03T08:58:08.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 52512 18.70 MB/sec warmup 80 sec latency 20.445 ms 2024-05-03T08:58:08.879 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:08 smithi032 ceph-mon[22403]: pgmap v280: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.5 MiB/s wr, 770 op/s 2024-05-03T08:58:09.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:08 smithi123 ceph-mon[24496]: pgmap v280: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.5 MiB/s wr, 770 op/s 2024-05-03T08:58:09.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 53314 18.85 MB/sec warmup 81 sec latency 25.714 ms 2024-05-03T08:58:10.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 53777 18.79 MB/sec warmup 82 sec latency 30.421 ms 2024-05-03T08:58:10.903 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:10 smithi032 ceph-mon[22403]: pgmap v281: 97 pgs: 97 active+clean; 287 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.3 MiB/s wr, 816 op/s 2024-05-03T08:58:11.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:10 smithi123 ceph-mon[24496]: pgmap v281: 97 pgs: 97 active+clean; 287 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.3 MiB/s wr, 816 op/s 2024-05-03T08:58:11.339 INFO:teuthology.orchestra.run.smithi032.stdout: 5 54319 18.65 MB/sec warmup 83 sec latency 22.644 ms 2024-05-03T08:58:12.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 54901 18.62 MB/sec warmup 84 sec latency 33.722 ms 2024-05-03T08:58:13.013 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:12 smithi032 ceph-mon[22403]: pgmap v282: 97 pgs: 97 active+clean; 288 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 828 op/s 2024-05-03T08:58:13.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:12 smithi123 ceph-mon[24496]: pgmap v282: 97 pgs: 97 active+clean; 288 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 828 op/s 2024-05-03T08:58:13.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 55667 18.56 MB/sec warmup 85 sec latency 31.952 ms 2024-05-03T08:58:14.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 56218 18.58 MB/sec warmup 86 sec latency 21.391 ms 2024-05-03T08:58:15.034 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:14 smithi032 ceph-mon[22403]: pgmap v283: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.2 MiB/s wr, 892 op/s 2024-05-03T08:58:15.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:14 smithi123 ceph-mon[24496]: pgmap v283: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.2 MiB/s wr, 892 op/s 2024-05-03T08:58:15.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 56999 18.73 MB/sec warmup 87 sec latency 21.502 ms 2024-05-03T08:58:16.041 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:15 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:58:16.041 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:15 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:58:16.041 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:15 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:58:16.083 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:15 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-03T08:58:16.083 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:15 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-03T08:58:16.083 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:15 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-03T08:58:16.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 57353 18.59 MB/sec warmup 88 sec latency 25.773 ms 2024-05-03T08:58:17.296 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:17 smithi032 ceph-mon[22403]: pgmap v284: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 925 op/s 2024-05-03T08:58:17.296 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:17 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:58:17.297 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:17 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:58:17.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 57897 18.47 MB/sec warmup 89 sec latency 42.246 ms 2024-05-03T08:58:17.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:17 smithi123 ceph-mon[24496]: pgmap v284: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 925 op/s 2024-05-03T08:58:17.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:17 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:58:17.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:17 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' 2024-05-03T08:58:18.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 58385 18.41 MB/sec warmup 90 sec latency 41.676 ms 2024-05-03T08:58:19.329 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:19 smithi032 ceph-mon[22403]: pgmap v285: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 867 op/s 2024-05-03T08:58:19.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 59087 18.37 MB/sec warmup 91 sec latency 32.793 ms 2024-05-03T08:58:19.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:19 smithi123 ceph-mon[24496]: pgmap v285: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 867 op/s 2024-05-03T08:58:20.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 59541 18.32 MB/sec warmup 92 sec latency 26.629 ms 2024-05-03T08:58:21.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 60226 18.41 MB/sec warmup 93 sec latency 25.056 ms 2024-05-03T08:58:21.364 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:21 smithi032 ceph-mon[22403]: pgmap v286: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.6 MiB/s wr, 882 op/s 2024-05-03T08:58:21.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:21 smithi123 ceph-mon[24496]: pgmap v286: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.6 MiB/s wr, 882 op/s 2024-05-03T08:58:22.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 60736 18.41 MB/sec warmup 94 sec latency 25.464 ms 2024-05-03T08:58:23.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 61069 18.27 MB/sec warmup 95 sec latency 40.367 ms 2024-05-03T08:58:23.385 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:23 smithi032 ceph-mon[22403]: pgmap v287: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 8.8 MiB/s wr, 747 op/s 2024-05-03T08:58:23.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:23 smithi123 ceph-mon[24496]: pgmap v287: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 8.8 MiB/s wr, 747 op/s 2024-05-03T08:58:24.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 61530 18.13 MB/sec warmup 96 sec latency 42.590 ms 2024-05-03T08:58:25.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 62053 18.10 MB/sec warmup 97 sec latency 29.481 ms 2024-05-03T08:58:25.398 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:25 smithi032 ceph-mon[22403]: pgmap v288: 97 pgs: 97 active+clean; 311 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.5 MiB/s wr, 876 op/s 2024-05-03T08:58:25.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:25 smithi123 ceph-mon[24496]: pgmap v288: 97 pgs: 97 active+clean; 311 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.5 MiB/s wr, 876 op/s 2024-05-03T08:58:26.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 62706 18.05 MB/sec warmup 98 sec latency 41.060 ms 2024-05-03T08:58:27.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 63213 18.03 MB/sec warmup 99 sec latency 22.817 ms 2024-05-03T08:58:27.409 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:27 smithi032 ceph-mon[22403]: pgmap v289: 97 pgs: 97 active+clean; 319 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 8.4 MiB/s wr, 782 op/s 2024-05-03T08:58:27.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:27 smithi123 ceph-mon[24496]: pgmap v289: 97 pgs: 97 active+clean; 319 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 8.4 MiB/s wr, 782 op/s 2024-05-03T08:58:28.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 63854 18.09 MB/sec warmup 100 sec latency 31.430 ms 2024-05-03T08:58:29.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 64316 18.09 MB/sec warmup 101 sec latency 28.354 ms 2024-05-03T08:58:29.441 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:29 smithi032 ceph-mon[22403]: pgmap v290: 97 pgs: 97 active+clean; 319 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.3 KiB/s rd, 8.0 MiB/s wr, 698 op/s 2024-05-03T08:58:29.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:29 smithi123 ceph-mon[24496]: pgmap v290: 97 pgs: 97 active+clean; 319 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.3 KiB/s rd, 8.0 MiB/s wr, 698 op/s 2024-05-03T08:58:30.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 64885 17.97 MB/sec warmup 102 sec latency 40.607 ms 2024-05-03T08:58:31.263 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:31 smithi032 ceph-mon[22403]: pgmap v291: 97 pgs: 97 active+clean; 322 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.2 MiB/s wr, 768 op/s 2024-05-03T08:58:31.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 65247 17.89 MB/sec warmup 103 sec latency 105.821 ms 2024-05-03T08:58:31.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:31 smithi123 ceph-mon[24496]: pgmap v291: 97 pgs: 97 active+clean; 322 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.2 MiB/s wr, 768 op/s 2024-05-03T08:58:32.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 65803 17.87 MB/sec warmup 104 sec latency 33.702 ms 2024-05-03T08:58:33.196 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:33 smithi032 ceph-mon[22403]: pgmap v292: 97 pgs: 97 active+clean; 326 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 8.0 MiB/s wr, 743 op/s 2024-05-03T08:58:33.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 66417 17.81 MB/sec warmup 105 sec latency 29.370 ms 2024-05-03T08:58:33.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:33 smithi123 ceph-mon[24496]: pgmap v292: 97 pgs: 97 active+clean; 326 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 8.0 MiB/s wr, 743 op/s 2024-05-03T08:58:34.340 INFO:teuthology.orchestra.run.smithi032.stdout: 5 66922 17.82 MB/sec warmup 106 sec latency 48.319 ms 2024-05-03T08:58:35.342 INFO:teuthology.orchestra.run.smithi032.stdout: 5 67531 17.85 MB/sec warmup 107 sec latency 26.184 ms 2024-05-03T08:58:35.385 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:35 smithi032 ceph-mon[22403]: pgmap v293: 97 pgs: 97 active+clean; 330 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.7 MiB/s wr, 803 op/s 2024-05-03T08:58:35.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:35 smithi123 ceph-mon[24496]: pgmap v293: 97 pgs: 97 active+clean; 330 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.7 MiB/s wr, 803 op/s 2024-05-03T08:58:36.343 INFO:teuthology.orchestra.run.smithi032.stdout: 5 67944 17.80 MB/sec warmup 108 sec latency 47.851 ms 2024-05-03T08:58:37.343 INFO:teuthology.orchestra.run.smithi032.stdout: 5 68403 17.69 MB/sec warmup 109 sec latency 38.961 ms 2024-05-03T08:58:37.407 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:37 smithi032 ceph-mon[22403]: pgmap v294: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 740 op/s 2024-05-03T08:58:37.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:37 smithi123 ceph-mon[24496]: pgmap v294: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 740 op/s 2024-05-03T08:58:38.343 INFO:teuthology.orchestra.run.smithi032.stdout: 5 68796 17.60 MB/sec warmup 110 sec latency 51.953 ms 2024-05-03T08:58:38.906 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -e sha1=f8e6d2d721bc96a93915917c59bc3d287d57263d -- bash -c 'ceph orch ps' 2024-05-03T08:58:39.343 INFO:teuthology.orchestra.run.smithi032.stdout: 5 69274 17.58 MB/sec warmup 111 sec latency 55.135 ms 2024-05-03T08:58:39.422 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:39 smithi032 ceph-mon[22403]: pgmap v295: 97 pgs: 97 active+clean; 337 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.7 MiB/s wr, 816 op/s 2024-05-03T08:58:39.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:39 smithi123 ceph-mon[24496]: pgmap v295: 97 pgs: 97 active+clean; 337 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.7 MiB/s wr, 816 op/s 2024-05-03T08:58:40.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 69868 17.51 MB/sec warmup 112 sec latency 49.747 ms 2024-05-03T08:58:41.343 INFO:teuthology.orchestra.run.smithi032.stdout: 5 70373 17.52 MB/sec warmup 113 sec latency 62.691 ms 2024-05-03T08:58:41.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:41 smithi123 ceph-mon[24496]: pgmap v296: 97 pgs: 97 active+clean; 341 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.5 MiB/s wr, 872 op/s 2024-05-03T08:58:41.486 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:41 smithi032 ceph-mon[22403]: pgmap v296: 97 pgs: 97 active+clean; 341 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.5 MiB/s wr, 872 op/s 2024-05-03T08:58:42.240 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:58:42.240 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:42 smithi032 ceph-mon[22403]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:58:42.343 INFO:teuthology.orchestra.run.smithi032.stdout: 5 71015 17.57 MB/sec warmup 114 sec latency 45.170 ms 2024-05-03T08:58:42.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/trash_purge_schedule"}]: dispatch 2024-05-03T08:58:42.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:42 smithi123 ceph-mon[24496]: from='mgr.14202 172.21.15.32:0/3084935705' entity='mgr.smithi032.umllay' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi032.umllay/mirror_snapshot_schedule"}]: dispatch 2024-05-03T08:58:42.909 INFO:teuthology.orchestra.run.smithi032.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-03T08:58:42.910 INFO:teuthology.orchestra.run.smithi032.stdout:alertmanager.smithi032 smithi032 *:9093,9094 running (7m) 116s ago 8m 24.3M - ba2b418f427c ddf53d5d27fc 2024-05-03T08:58:42.910 INFO:teuthology.orchestra.run.smithi032.stdout:crash.smithi032 smithi032 running (8m) 116s ago 8m 6409k - 17.2.0 e1d6a67b021e 38dc8227bd42 2024-05-03T08:58:42.910 INFO:teuthology.orchestra.run.smithi032.stdout:crash.smithi123 smithi123 running (7m) 118s ago 7m 6405k - 17.2.0 e1d6a67b021e 2bce696b8db8 2024-05-03T08:58:42.910 INFO:teuthology.orchestra.run.smithi032.stdout:grafana.smithi032 smithi032 *:3000 running (6m) 116s ago 8m 54.4M - 8.3.5 dad864ee21e9 81ca1cc63b54 2024-05-03T08:58:42.910 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foofs.smithi032.axmmtz smithi032 running (2m) 116s ago 2m 15.8M - 17.2.0 e1d6a67b021e 0c91c3379b95 2024-05-03T08:58:42.910 INFO:teuthology.orchestra.run.smithi032.stdout:mds.foofs.smithi123.gtrlts smithi123 running (2m) 118s ago 2m 13.4M - 17.2.0 e1d6a67b021e b24c9351f251 2024-05-03T08:58:42.910 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.smithi032.umllay smithi032 *:9283 running (9m) 116s ago 9m 460M - 17.2.0 e1d6a67b021e 537e53955aba 2024-05-03T08:58:42.910 INFO:teuthology.orchestra.run.smithi032.stdout:mgr.smithi123.xrxinh smithi123 *:8443,9283 running (7m) 118s ago 7m 412M - 17.2.0 e1d6a67b021e e7499f622d47 2024-05-03T08:58:42.910 INFO:teuthology.orchestra.run.smithi032.stdout:mon.smithi032 smithi032 running (9m) 116s ago 9m 48.2M 2048M 17.2.0 e1d6a67b021e 7e5da84a501e 2024-05-03T08:58:42.910 INFO:teuthology.orchestra.run.smithi032.stdout:mon.smithi123 smithi123 running (7m) 118s ago 7m 39.0M 2048M 17.2.0 e1d6a67b021e 3a32332bb758 2024-05-03T08:58:42.910 INFO:teuthology.orchestra.run.smithi032.stdout:nfs.foo.0.0.smithi032.myyxfz smithi032 *:2049 running (2m) 116s ago 2m 53.1M - 4.0 e1d6a67b021e adc1bba0e7ed 2024-05-03T08:58:42.910 INFO:teuthology.orchestra.run.smithi032.stdout:nfs.foo.1.0.smithi123.dkofzl smithi123 *:2049 running (2m) 118s ago 2m 52.9M - 4.0 e1d6a67b021e f5ab76e466b3 2024-05-03T08:58:42.910 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.smithi032 smithi032 *:9100 running (8m) 116s ago 8m 17.0M - 1dbe0e931976 ae21c41038e9 2024-05-03T08:58:42.910 INFO:teuthology.orchestra.run.smithi032.stdout:node-exporter.smithi123 smithi123 *:9100 running (7m) 118s ago 7m 15.9M - 1dbe0e931976 dabbb678f527 2024-05-03T08:58:42.911 INFO:teuthology.orchestra.run.smithi032.stdout:osd.0 smithi032 running (6m) 116s ago 6m 43.7M 3217M 17.2.0 e1d6a67b021e 167471b12229 2024-05-03T08:58:42.911 INFO:teuthology.orchestra.run.smithi032.stdout:osd.1 smithi032 running (6m) 116s ago 6m 46.2M 3217M 17.2.0 e1d6a67b021e 21b9c0b15d6f 2024-05-03T08:58:42.911 INFO:teuthology.orchestra.run.smithi032.stdout:osd.2 smithi032 running (6m) 116s ago 6m 39.0M 3217M 17.2.0 e1d6a67b021e 6dfd926898e3 2024-05-03T08:58:42.911 INFO:teuthology.orchestra.run.smithi032.stdout:osd.3 smithi032 running (5m) 116s ago 5m 40.6M 3217M 17.2.0 e1d6a67b021e 947099fbe826 2024-05-03T08:58:42.911 INFO:teuthology.orchestra.run.smithi032.stdout:osd.4 smithi123 running (5m) 118s ago 5m 40.6M 3984M 17.2.0 e1d6a67b021e 150ddc5685f9 2024-05-03T08:58:42.911 INFO:teuthology.orchestra.run.smithi032.stdout:osd.5 smithi123 running (4m) 118s ago 4m 42.0M 3984M 17.2.0 e1d6a67b021e 72ba92ce4b53 2024-05-03T08:58:42.911 INFO:teuthology.orchestra.run.smithi032.stdout:osd.6 smithi123 running (4m) 118s ago 4m 42.0M 3984M 17.2.0 e1d6a67b021e 2bc752f6bfc5 2024-05-03T08:58:42.911 INFO:teuthology.orchestra.run.smithi032.stdout:osd.7 smithi123 running (4m) 118s ago 4m 41.7M 3984M 17.2.0 e1d6a67b021e 40d54616bfbe 2024-05-03T08:58:42.911 INFO:teuthology.orchestra.run.smithi032.stdout:prometheus.smithi032 smithi032 *:9095 running (7m) 116s ago 7m 62.5M - 514e6a882f6e 6763db48c356 2024-05-03T08:58:43.243 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:43 smithi032 ceph-mon[22403]: pgmap v297: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.9 MiB/s wr, 789 op/s 2024-05-03T08:58:43.343 INFO:teuthology.orchestra.run.smithi032.stdout: 5 71450 17.54 MB/sec warmup 115 sec latency 23.999 ms 2024-05-03T08:58:43.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:43 smithi123 ceph-mon[24496]: pgmap v297: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.9 MiB/s wr, 789 op/s 2024-05-03T08:58:44.096 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -e sha1=f8e6d2d721bc96a93915917c59bc3d287d57263d -- bash -c 'ceph orch upgrade status' 2024-05-03T08:58:44.256 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:44 smithi032 ceph-mon[22403]: from='client.24427 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:58:44.343 INFO:teuthology.orchestra.run.smithi032.stdout: 5 71949 17.45 MB/sec warmup 116 sec latency 46.016 ms 2024-05-03T08:58:44.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:44 smithi123 ceph-mon[24496]: from='client.24427 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:58:45.267 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:45 smithi032 ceph-mon[22403]: pgmap v298: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 827 op/s 2024-05-03T08:58:45.343 INFO:teuthology.orchestra.run.smithi032.stdout: 5 72476 17.41 MB/sec warmup 117 sec latency 23.854 ms 2024-05-03T08:58:45.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:45 smithi123 ceph-mon[24496]: pgmap v298: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 827 op/s 2024-05-03T08:58:46.343 INFO:teuthology.orchestra.run.smithi032.stdout: 5 73124 17.39 MB/sec warmup 118 sec latency 29.468 ms 2024-05-03T08:58:47.263 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-05-03T08:58:47.263 INFO:teuthology.orchestra.run.smithi032.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:f8e6d2d721bc96a93915917c59bc3d287d57263d", 2024-05-03T08:58:47.263 INFO:teuthology.orchestra.run.smithi032.stdout: "in_progress": true, 2024-05-03T08:58:47.264 INFO:teuthology.orchestra.run.smithi032.stdout: "services_complete": [], 2024-05-03T08:58:47.264 INFO:teuthology.orchestra.run.smithi032.stdout: "progress": "", 2024-05-03T08:58:47.264 INFO:teuthology.orchestra.run.smithi032.stdout: "message": "Error: UPGRADE_FAILED_PULL: Upgrade: failed to pull target image" 2024-05-03T08:58:47.264 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-05-03T08:58:47.280 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:47 smithi032 ceph-mon[22403]: pgmap v299: 97 pgs: 97 active+clean; 334 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.6 MiB/s wr, 753 op/s 2024-05-03T08:58:47.343 INFO:teuthology.orchestra.run.smithi032.stdout: 5 73740 17.37 MB/sec warmup 119 sec latency 30.377 ms 2024-05-03T08:58:47.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:47 smithi123 ceph-mon[24496]: pgmap v299: 97 pgs: 97 active+clean; 334 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.6 MiB/s wr, 753 op/s 2024-05-03T08:58:48.251 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -e sha1=f8e6d2d721bc96a93915917c59bc3d287d57263d -- bash -c 'ceph health detail' 2024-05-03T08:58:48.293 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:48 smithi032 ceph-mon[22403]: from='client.24429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:58:48.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:48 smithi123 ceph-mon[24496]: from='client.24429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-03T08:58:49.298 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:49 smithi032 ceph-mon[22403]: pgmap v300: 97 pgs: 97 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 8.2 MiB/s wr, 830 op/s 2024-05-03T08:58:49.344 INFO:teuthology.orchestra.run.smithi032.stdout: 5 74913 23.10 MB/sec execute 1 sec latency 26.667 ms 2024-05-03T08:58:49.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:49 smithi123 ceph-mon[24496]: pgmap v300: 97 pgs: 97 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 8.2 MiB/s wr, 830 op/s 2024-05-03T08:58:50.344 INFO:teuthology.orchestra.run.smithi032.stdout: 5 75404 15.42 MB/sec execute 2 sec latency 25.329 ms 2024-05-03T08:58:51.312 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:51 smithi032 ceph-mon[22403]: pgmap v301: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.3 MiB/s wr, 765 op/s 2024-05-03T08:58:51.344 INFO:teuthology.orchestra.run.smithi032.stdout: 5 75817 11.95 MB/sec execute 3 sec latency 49.118 ms 2024-05-03T08:58:51.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:51 smithi123 ceph-mon[24496]: pgmap v301: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.3 MiB/s wr, 765 op/s 2024-05-03T08:58:52.056 INFO:teuthology.orchestra.run.smithi032.stdout:HEALTH_WARN Upgrade: failed to pull target image 2024-05-03T08:58:52.057 INFO:teuthology.orchestra.run.smithi032.stdout:[WRN] UPGRADE_FAILED_PULL: Upgrade: failed to pull target image 2024-05-03T08:58:52.057 INFO:teuthology.orchestra.run.smithi032.stdout: host smithi032 `cephadm pull` failed: cephadm exited with an error code: 1, stderr: Pulling container image quay.ceph.io/ceph-ci/ceph:f8e6d2d721bc96a93915917c59bc3d287d57263d... 2024-05-03T08:58:52.057 INFO:teuthology.orchestra.run.smithi032.stdout:Non-zero exit code 125 from /bin/podman pull quay.ceph.io/ceph-ci/ceph:f8e6d2d721bc96a93915917c59bc3d287d57263d 2024-05-03T08:58:52.057 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:f8e6d2d721bc96a93915917c59bc3d287d57263d... 2024-05-03T08:58:52.057 INFO:teuthology.orchestra.run.smithi032.stdout:/bin/podman: stderr Error: initializing source docker://quay.ceph.io/ceph-ci/ceph:f8e6d2d721bc96a93915917c59bc3d287d57263d: reading manifest f8e6d2d721bc96a93915917c59bc3d287d57263d in quay.ceph.io/ceph-ci/ceph: manifest unknown 2024-05-03T08:58:52.057 INFO:teuthology.orchestra.run.smithi032.stdout:ERROR: Failed command: /bin/podman pull quay.ceph.io/ceph-ci/ceph:f8e6d2d721bc96a93915917c59bc3d287d57263d 2024-05-03T08:58:52.321 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:52 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/919365701' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-03T08:58:52.344 INFO:teuthology.orchestra.run.smithi032.stdout: 5 76277 12.52 MB/sec execute 4 sec latency 37.022 ms 2024-05-03T08:58:52.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:52 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/919365701' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-03T08:58:52.951 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -e sha1=f8e6d2d721bc96a93915917c59bc3d287d57263d -- bash -c 'ceph versions' 2024-05-03T08:58:53.325 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:53 smithi032 ceph-mon[22403]: pgmap v302: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 8.0 MiB/s wr, 702 op/s 2024-05-03T08:58:53.344 INFO:teuthology.orchestra.run.smithi032.stdout: 5 76944 12.63 MB/sec execute 5 sec latency 29.111 ms 2024-05-03T08:58:53.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:53 smithi123 ceph-mon[24496]: pgmap v302: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 8.0 MiB/s wr, 702 op/s 2024-05-03T08:58:54.344 INFO:teuthology.orchestra.run.smithi032.stdout: 5 77399 13.07 MB/sec execute 6 sec latency 29.695 ms 2024-05-03T08:58:55.344 INFO:teuthology.orchestra.run.smithi032.stdout: 5 77969 14.58 MB/sec execute 7 sec latency 30.817 ms 2024-05-03T08:58:55.467 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:55 smithi032 ceph-mon[22403]: pgmap v303: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 8.2 MiB/s wr, 828 op/s 2024-05-03T08:58:55.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:55 smithi123 ceph-mon[24496]: pgmap v303: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 8.2 MiB/s wr, 828 op/s 2024-05-03T08:58:56.268 INFO:teuthology.orchestra.run.smithi032.stdout:{ 2024-05-03T08:58:56.268 INFO:teuthology.orchestra.run.smithi032.stdout: "mon": { 2024-05-03T08:58:56.268 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-03T08:58:56.268 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-05-03T08:58:56.269 INFO:teuthology.orchestra.run.smithi032.stdout: "mgr": { 2024-05-03T08:58:56.269 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-03T08:58:56.269 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-05-03T08:58:56.269 INFO:teuthology.orchestra.run.smithi032.stdout: "osd": { 2024-05-03T08:58:56.269 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-03T08:58:56.269 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-05-03T08:58:56.269 INFO:teuthology.orchestra.run.smithi032.stdout: "mds": { 2024-05-03T08:58:56.269 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-03T08:58:56.269 INFO:teuthology.orchestra.run.smithi032.stdout: }, 2024-05-03T08:58:56.269 INFO:teuthology.orchestra.run.smithi032.stdout: "overall": { 2024-05-03T08:58:56.269 INFO:teuthology.orchestra.run.smithi032.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-05-03T08:58:56.269 INFO:teuthology.orchestra.run.smithi032.stdout: } 2024-05-03T08:58:56.269 INFO:teuthology.orchestra.run.smithi032.stdout:} 2024-05-03T08:58:56.344 INFO:teuthology.orchestra.run.smithi032.stdout: 5 78560 15.04 MB/sec execute 8 sec latency 23.558 ms 2024-05-03T08:58:56.974 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:56 smithi032 ceph-mon[22403]: pgmap v304: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 685 op/s 2024-05-03T08:58:56.974 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:56 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/1378337533' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-03T08:58:57.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:56 smithi123 ceph-mon[24496]: pgmap v304: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 685 op/s 2024-05-03T08:58:57.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:56 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/1378337533' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-03T08:58:57.344 INFO:teuthology.orchestra.run.smithi032.stdout: 5 79008 14.10 MB/sec execute 9 sec latency 40.282 ms 2024-05-03T08:58:57.834 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -e sha1=f8e6d2d721bc96a93915917c59bc3d287d57263d -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-05-03T08:58:58.344 INFO:teuthology.orchestra.run.smithi032.stdout: 5 79507 13.68 MB/sec execute 10 sec latency 32.046 ms 2024-05-03T08:58:58.990 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:58:58 smithi032 ceph-mon[22403]: pgmap v305: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 815 op/s 2024-05-03T08:58:59.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:58:58 smithi123 ceph-mon[24496]: pgmap v305: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 815 op/s 2024-05-03T08:58:59.344 INFO:teuthology.orchestra.run.smithi032.stdout: 5 80098 13.89 MB/sec execute 11 sec latency 32.799 ms 2024-05-03T08:59:00.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 80620 13.50 MB/sec execute 12 sec latency 34.396 ms 2024-05-03T08:59:01.025 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:59:00 smithi032 ceph-mon[22403]: pgmap v306: 97 pgs: 97 active+clean; 329 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 6.9 MiB/s wr, 617 op/s 2024-05-03T08:59:01.162 INFO:teuthology.orchestra.run.smithi032.stdout:true 2024-05-03T08:59:01.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:59:00 smithi123 ceph-mon[24496]: pgmap v306: 97 pgs: 97 active+clean; 329 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 6.9 MiB/s wr, 617 op/s 2024-05-03T08:59:01.344 INFO:teuthology.orchestra.run.smithi032.stdout: 5 81114 14.05 MB/sec execute 13 sec latency 34.001 ms 2024-05-03T08:59:02.033 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:59:01 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/2524005944' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-03T08:59:02.065 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -e sha1=f8e6d2d721bc96a93915917c59bc3d287d57263d -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-05-03T08:59:02.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:59:01 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/2524005944' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-03T08:59:02.344 INFO:teuthology.orchestra.run.smithi032.stdout: 5 81616 14.44 MB/sec execute 14 sec latency 37.594 ms 2024-05-03T08:59:03.052 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:59:02 smithi032 ceph-mon[22403]: pgmap v307: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 7.6 MiB/s wr, 632 op/s 2024-05-03T08:59:03.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:59:02 smithi123 ceph-mon[24496]: pgmap v307: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 7.6 MiB/s wr, 632 op/s 2024-05-03T08:59:03.344 INFO:teuthology.orchestra.run.smithi032.stdout: 5 81987 14.27 MB/sec execute 15 sec latency 34.512 ms 2024-05-03T08:59:04.344 INFO:teuthology.orchestra.run.smithi032.stdout: 5 82282 13.62 MB/sec execute 16 sec latency 32.249 ms 2024-05-03T08:59:05.086 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:59:04 smithi032 ceph-mon[22403]: pgmap v308: 97 pgs: 97 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.5 MiB/s wr, 807 op/s 2024-05-03T08:59:05.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:59:04 smithi123 ceph-mon[24496]: pgmap v308: 97 pgs: 97 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.5 MiB/s wr, 807 op/s 2024-05-03T08:59:05.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 82702 13.19 MB/sec execute 17 sec latency 34.846 ms 2024-05-03T08:59:06.099 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:59:05 smithi032 ceph-mon[22403]: from='client.? 172.21.15.32:0/3988295595' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-03T08:59:06.227 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:59:05 smithi123 ceph-mon[24496]: from='client.? 172.21.15.32:0/3988295595' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-03T08:59:06.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 83163 13.12 MB/sec execute 18 sec latency 43.305 ms 2024-05-03T08:59:06.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-03T08:59:06.430 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_f8e6d2d721bc96a93915917c59bc3d287d57263d/qa/tasks/cephadm.py", line 1475, in shell _shell( File "/home/teuthworker/src/git.ceph.com_ceph-c_f8e6d2d721bc96a93915917c59bc3d287d57263d/qa/tasks/cephadm.py", line 110, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi032 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -e sha1=f8e6d2d721bc96a93915917c59bc3d287d57263d -- bash -c \'ceph versions | jq -e \'"\'"\'.overall | keys\'"\'"\' | grep $sha1\'' 2024-05-03T08:59:06.646 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=b75a8212441840eea39e7eba82a24b72 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_f8e6d2d721bc96a93915917c59bc3d287d57263d/qa/tasks/cephadm.py", line 1475, in shell _shell( File "/home/teuthworker/src/git.ceph.com_ceph-c_f8e6d2d721bc96a93915917c59bc3d287d57263d/qa/tasks/cephadm.py", line 110, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi032 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -e sha1=f8e6d2d721bc96a93915917c59bc3d287d57263d -- bash -c \'ceph versions | jq -e \'"\'"\'.overall | keys\'"\'"\' | grep $sha1\'' 2024-05-03T08:59:06.649 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-05-03T08:59:06.660 INFO:tasks.cephadm:Teardown begin 2024-05-03T08:59:06.660 DEBUG:teuthology.orchestra.run.smithi032:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-03T08:59:06.699 DEBUG:teuthology.orchestra.run.smithi123:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-03T08:59:06.727 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-05-03T08:59:06.727 DEBUG:teuthology.orchestra.run.smithi032:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-05-03T08:59:06.761 DEBUG:teuthology.orchestra.run.smithi123:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-05-03T08:59:06.782 INFO:tasks.cephadm:Stopping all daemons... 2024-05-03T08:59:06.782 INFO:tasks.cephadm.mon.smithi032:Stopping mon.smithi032... 2024-05-03T08:59:06.782 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-df63fec2-0929-11ef-bc97-c7b262605968@mon.smithi032 2024-05-03T08:59:06.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:59:06 smithi123 ceph-mon[24496]: pgmap v309: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.5 MiB/s wr, 709 op/s 2024-05-03T08:59:07.107 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:59:06 smithi032 ceph-mon[22403]: pgmap v309: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.5 MiB/s wr, 709 op/s 2024-05-03T08:59:07.107 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:59:06 smithi032 systemd[1]: Stopping Ceph mon.smithi032 for df63fec2-0929-11ef-bc97-c7b262605968... 2024-05-03T08:59:07.107 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:59:06 smithi032 bash[55445]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon.smithi032" found: no such container 2024-05-03T08:59:07.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 83806 13.22 MB/sec execute 19 sec latency 34.836 ms 2024-05-03T08:59:07.357 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:59:07 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi032[22399]: 2024-05-03T08:59:07.272+0000 7f592614d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi032 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-03T08:59:07.357 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:59:07 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi032[22399]: 2024-05-03T08:59:07.272+0000 7f592614d700 -1 mon.smithi032@0(leader) e2 *** Got Signal Terminated *** 2024-05-03T08:59:07.608 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:59:07 smithi032 podman[55455]: 2024-05-03 08:59:07.387410226 +0000 UTC m=+0.469054911 container died 7e5da84a501eb3ff7337cde84c7f3667e36b6b4ce6f199f91b9b974421ca3ff8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi032, name=centos-stream, maintainer=Guillaume Abrioux , release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.openshift.expose-services=, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, ceph=True, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-05-03T08:59:08.113 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:59:07 smithi032 podman[55455]: 2024-05-03 08:59:07.972374543 +0000 UTC m=+1.054019224 container cleanup 7e5da84a501eb3ff7337cde84c7f3667e36b6b4ce6f199f91b9b974421ca3ff8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi032, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, io.openshift.expose-services=, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, architecture=x86_64, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, version=8, io.buildah.version=1.19.8, name=centos-stream, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git) 2024-05-03T08:59:08.113 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:59:07 smithi032 bash[55455]: ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi032 2024-05-03T08:59:08.114 INFO:journalctl@ceph.mon.smithi032.smithi032.stdout:May 03 08:59:08 smithi032 bash[55481]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon.smithi032" found: no such container 2024-05-03T08:59:08.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84377 13.31 MB/sec execute 20 sec latency 32.934 ms 2024-05-03T08:59:08.383 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@mon.smithi032.service' 2024-05-03T08:59:08.507 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-03T08:59:08.508 INFO:tasks.cephadm.mon.smithi032:Stopped mon.smithi032 2024-05-03T08:59:08.508 INFO:tasks.cephadm.mon.smithi123:Stopping mon.smithi123... 2024-05-03T08:59:08.508 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-df63fec2-0929-11ef-bc97-c7b262605968@mon.smithi123 2024-05-03T08:59:08.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:59:08 smithi123 systemd[1]: Stopping Ceph mon.smithi123 for df63fec2-0929-11ef-bc97-c7b262605968... 2024-05-03T08:59:08.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:59:08 smithi123 bash[40621]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon.smithi123" found: no such container 2024-05-03T08:59:09.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 13.41 MB/sec execute 21 sec latency 318.283 ms 2024-05-03T08:59:09.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:59:09 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi123[24486]: 2024-05-03T08:59:09.023+0000 7f362b270700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi123 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-03T08:59:09.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:59:09 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi123[24486]: 2024-05-03T08:59:09.023+0000 7f362b270700 -1 mon.smithi123@1(peon) e2 *** Got Signal Terminated *** 2024-05-03T08:59:09.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:59:09 smithi123 podman[40634]: 2024-05-03 08:59:09.064640811 +0000 UTC m=+0.479590718 container died 3a32332bb758801c97a94f2bc07f9bdb2def9c0c75941f99a245203809b759e4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi123, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, name=centos-stream, vcs-type=git, release=754) 2024-05-03T08:59:09.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:59:09 smithi123 podman[40634]: 2024-05-03 08:59:09.540245509 +0000 UTC m=+0.955195424 container cleanup 3a32332bb758801c97a94f2bc07f9bdb2def9c0c75941f99a245203809b759e4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi123, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , distribution-scope=public, version=8, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, GIT_BRANCH=HEAD, name=centos-stream, architecture=x86_64) 2024-05-03T08:59:09.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:59:09 smithi123 bash[40634]: ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon-smithi123 2024-05-03T08:59:09.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:May 03 08:59:09 smithi123 bash[40662]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-mon.smithi123" found: no such container 2024-05-03T08:59:09.823 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@mon.smithi123.service' 2024-05-03T08:59:09.869 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-03T08:59:09.869 INFO:tasks.cephadm.mon.smithi123:Stopped mon.smithi123 2024-05-03T08:59:09.870 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-05-03T08:59:09.870 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.0 2024-05-03T08:59:10.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 12.80 MB/sec execute 22 sec latency 1318.310 ms 2024-05-03T08:59:10.388 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:59:09 smithi032 systemd[1]: Stopping Ceph osd.0 for df63fec2-0929-11ef-bc97-c7b262605968... 2024-05-03T08:59:10.388 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:59:09 smithi032 bash[55530]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd.0" found: no such container 2024-05-03T08:59:10.388 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:59:10 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-0[32803]: 2024-05-03T08:59:10.131+0000 7f9861f40700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-03T08:59:10.388 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:59:10 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-0[32803]: 2024-05-03T08:59:10.131+0000 7f9861f40700 -1 osd.0 69 *** Got signal Terminated *** 2024-05-03T08:59:10.388 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:59:10 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-0[32803]: 2024-05-03T08:59:10.131+0000 7f9861f40700 -1 osd.0 69 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-03T08:59:11.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 12.24 MB/sec execute 23 sec latency 2318.360 ms 2024-05-03T08:59:12.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 11.73 MB/sec execute 24 sec latency 3318.402 ms 2024-05-03T08:59:13.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 11.26 MB/sec execute 25 sec latency 4318.452 ms 2024-05-03T08:59:14.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 10.83 MB/sec execute 26 sec latency 5318.517 ms 2024-05-03T08:59:15.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 10.43 MB/sec execute 27 sec latency 6318.546 ms 2024-05-03T08:59:16.031 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:59:15 smithi032 podman[55543]: 2024-05-03 08:59:15.74106571 +0000 UTC m=+5.750263554 container died 167471b1222912c4f0dad16b958b9fd869df85e5dc004d092785a3d502407400 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-0, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, version=8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, release=754, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_BRANCH=HEAD, name=centos-stream, io.buildah.version=1.19.8) 2024-05-03T08:59:16.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 10.06 MB/sec execute 28 sec latency 7318.572 ms 2024-05-03T08:59:16.388 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:59:16 smithi032 podman[55543]: 2024-05-03 08:59:16.253937437 +0000 UTC m=+6.263135278 container cleanup 167471b1222912c4f0dad16b958b9fd869df85e5dc004d092785a3d502407400 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-0, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, RELEASE=HEAD, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, ceph=True, release=754, vendor=Red Hat, Inc., io.openshift.expose-services=, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container) 2024-05-03T08:59:16.388 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:59:16 smithi032 bash[55543]: ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-0 2024-05-03T08:59:16.712 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:59:16 smithi032 podman[55560]: 2024-05-03 08:59:16.421961405 +0000 UTC m=+0.676602424 container remove 167471b1222912c4f0dad16b958b9fd869df85e5dc004d092785a3d502407400 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-0, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, distribution-scope=public, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, name=centos-stream, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8) 2024-05-03T08:59:16.712 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:59:16 smithi032 bash[55573]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd.0" found: no such container 2024-05-03T08:59:17.026 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:59:16 smithi032 podman[55634]: 2024-05-03 08:59:16.615464944 +0000 UTC m=+0.019381402 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-03T08:59:17.029 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:59:16 smithi032 podman[55634]: 2024-05-03 08:59:16.740939864 +0000 UTC m=+0.144856322 container create b85660a1ad633e7a0b66daeaf1d9c89a0cc5e3f52272eb6eca50edacbe6b74f5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-0-deactivate, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, version=8, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, RELEASE=HEAD, distribution-scope=public) 2024-05-03T08:59:17.029 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:59:17 smithi032 podman[55634]: 2024-05-03 08:59:17.025707608 +0000 UTC m=+0.429624066 container init b85660a1ad633e7a0b66daeaf1d9c89a0cc5e3f52272eb6eca50edacbe6b74f5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-0-deactivate, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , name=centos-stream, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, ceph=True, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8) 2024-05-03T08:59:17.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 9.71 MB/sec execute 29 sec latency 8318.602 ms 2024-05-03T08:59:17.388 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:59:17 smithi032 podman[55634]: 2024-05-03 08:59:17.031425721 +0000 UTC m=+0.435342186 container start b85660a1ad633e7a0b66daeaf1d9c89a0cc5e3f52272eb6eca50edacbe6b74f5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-0-deactivate, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, distribution-scope=public, architecture=x86_64, vcs-type=git, name=centos-stream, com.redhat.component=centos-stream-container, release=754, maintainer=Guillaume Abrioux , io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, vendor=Red Hat, Inc., GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, ceph=True) 2024-05-03T08:59:17.388 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:59:17 smithi032 podman[55634]: 2024-05-03 08:59:17.13403741 +0000 UTC m=+0.537953876 container attach b85660a1ad633e7a0b66daeaf1d9c89a0cc5e3f52272eb6eca50edacbe6b74f5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-0-deactivate, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, release=754, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, vendor=Red Hat, Inc., RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , architecture=x86_64, GIT_BRANCH=HEAD, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-03T08:59:17.755 INFO:journalctl@ceph.osd.0.smithi032.stdout:May 03 08:59:17 smithi032 podman[55634]: 2024-05-03 08:59:17.39795999 +0000 UTC m=+0.801876438 container died b85660a1ad633e7a0b66daeaf1d9c89a0cc5e3f52272eb6eca50edacbe6b74f5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-0-deactivate, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, release=754, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, GIT_CLEAN=True, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, maintainer=Guillaume Abrioux , architecture=x86_64, GIT_BRANCH=HEAD, io.openshift.expose-services=) 2024-05-03T08:59:18.124 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.0.service' 2024-05-03T08:59:18.161 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-03T08:59:18.161 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-05-03T08:59:18.162 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-05-03T08:59:18.162 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.1 2024-05-03T08:59:18.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 9.39 MB/sec execute 30 sec latency 9318.625 ms 2024-05-03T08:59:18.388 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:59:18 smithi032 systemd[1]: Stopping Ceph osd.1 for df63fec2-0929-11ef-bc97-c7b262605968... 2024-05-03T08:59:18.388 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:59:18 smithi032 bash[55712]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd.1" found: no such container 2024-05-03T08:59:18.888 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:59:18 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-1[36879]: 2024-05-03T08:59:18.453+0000 7f799000b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-03T08:59:18.888 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:59:18 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-1[36879]: 2024-05-03T08:59:18.453+0000 7f799000b700 -1 osd.1 69 *** Got signal Terminated *** 2024-05-03T08:59:18.888 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:59:18 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-1[36879]: 2024-05-03T08:59:18.453+0000 7f799000b700 -1 osd.1 69 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-03T08:59:19.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 9.08 MB/sec execute 31 sec latency 10318.652 ms 2024-05-03T08:59:20.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 8.80 MB/sec execute 32 sec latency 11318.678 ms 2024-05-03T08:59:21.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 8.53 MB/sec execute 33 sec latency 12318.728 ms 2024-05-03T08:59:22.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 8.28 MB/sec execute 34 sec latency 13318.776 ms 2024-05-03T08:59:23.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 8.04 MB/sec execute 35 sec latency 14318.803 ms 2024-05-03T08:59:24.236 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:59:23 smithi032 podman[55726]: 2024-05-03 08:59:23.915851245 +0000 UTC m=+5.609371944 container died 21b9c0b15d6f25cfb84289b74db08461d37ec15c3cffcd00ecb518f9191d0d7d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-1, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, version=8, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, RELEASE=HEAD, architecture=x86_64, vcs-type=git, vendor=Red Hat, Inc., GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream) 2024-05-03T08:59:24.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 7.82 MB/sec execute 36 sec latency 15318.851 ms 2024-05-03T08:59:24.492 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:59:24 smithi032 podman[55726]: 2024-05-03 08:59:24.265541172 +0000 UTC m=+5.959061871 container cleanup 21b9c0b15d6f25cfb84289b74db08461d37ec15c3cffcd00ecb518f9191d0d7d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-1, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, version=8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, GIT_BRANCH=HEAD, architecture=x86_64, GIT_CLEAN=True, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, ceph=True, name=centos-stream, com.redhat.component=centos-stream-container) 2024-05-03T08:59:24.492 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:59:24 smithi032 bash[55726]: ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-1 2024-05-03T08:59:24.492 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:59:24 smithi032 podman[55747]: 2024-05-03 08:59:24.425125483 +0000 UTC m=+0.500139406 container remove 21b9c0b15d6f25cfb84289b74db08461d37ec15c3cffcd00ecb518f9191d0d7d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-1, io.openshift.tags=base centos centos-stream, distribution-scope=public, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, name=centos-stream, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.buildah.version=1.19.8, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, GIT_CLEAN=True) 2024-05-03T08:59:24.492 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:59:24 smithi032 bash[55761]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd.1" found: no such container 2024-05-03T08:59:24.963 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:59:24 smithi032 podman[55825]: 2024-05-03 08:59:24.609575267 +0000 UTC m=+0.019807503 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-03T08:59:24.963 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:59:24 smithi032 podman[55825]: 2024-05-03 08:59:24.734349465 +0000 UTC m=+0.144581686 container create 6d6fe60f753f0e15959a26a237bfc69aaba8ad1ee121c3d4ecb73bf73b081633 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-1-deactivate, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, RELEASE=HEAD, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, GIT_BRANCH=HEAD, name=centos-stream, GIT_CLEAN=True, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, vendor=Red Hat, Inc., distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, version=8, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-05-03T08:59:24.963 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:59:24 smithi032 podman[55825]: 2024-05-03 08:59:24.953610237 +0000 UTC m=+0.363842460 container init 6d6fe60f753f0e15959a26a237bfc69aaba8ad1ee121c3d4ecb73bf73b081633 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-1-deactivate, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, name=centos-stream, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, release=754, maintainer=Guillaume Abrioux , ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, version=8, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-05-03T08:59:24.963 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:59:24 smithi032 podman[55825]: 2024-05-03 08:59:24.961834398 +0000 UTC m=+0.372066628 container start 6d6fe60f753f0e15959a26a237bfc69aaba8ad1ee121c3d4ecb73bf73b081633 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, GIT_CLEAN=True, version=8, release=754, GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.buildah.version=1.19.8, name=centos-stream, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0) 2024-05-03T08:59:25.313 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:59:25 smithi032 podman[55825]: 2024-05-03 08:59:25.061963884 +0000 UTC m=+0.472196120 container attach 6d6fe60f753f0e15959a26a237bfc69aaba8ad1ee121c3d4ecb73bf73b081633 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-1-deactivate, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, vcs-type=git, GIT_CLEAN=True, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, name=centos-stream, ceph=True, distribution-scope=public) 2024-05-03T08:59:25.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 7.61 MB/sec execute 37 sec latency 16318.910 ms 2024-05-03T08:59:25.638 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:59:25 smithi032 podman[55825]: 2024-05-03 08:59:25.312632032 +0000 UTC m=+0.722864254 container died 6d6fe60f753f0e15959a26a237bfc69aaba8ad1ee121c3d4ecb73bf73b081633 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-1-deactivate, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, version=8, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, GIT_CLEAN=True, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, distribution-scope=public, architecture=x86_64, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-05-03T08:59:25.957 INFO:journalctl@ceph.osd.1.smithi032.stdout:May 03 08:59:25 smithi032 podman[55851]: 2024-05-03 08:59:25.915247489 +0000 UTC m=+0.596566304 container remove 6d6fe60f753f0e15959a26a237bfc69aaba8ad1ee121c3d4ecb73bf73b081633 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-1-deactivate, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, vcs-type=git, release=754, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, architecture=x86_64, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.openshift.expose-services=) 2024-05-03T08:59:25.963 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.1.service' 2024-05-03T08:59:26.000 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-03T08:59:26.000 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-05-03T08:59:26.000 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-05-03T08:59:26.001 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.2 2024-05-03T08:59:26.260 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:59:26 smithi032 systemd[1]: Stopping Ceph osd.2 for df63fec2-0929-11ef-bc97-c7b262605968... 2024-05-03T08:59:26.260 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:59:26 smithi032 bash[55903]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd.2" found: no such container 2024-05-03T08:59:26.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 7.41 MB/sec execute 38 sec latency 17318.953 ms 2024-05-03T08:59:26.638 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:59:26 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-2[40608]: 2024-05-03T08:59:26.258+0000 7f1d3fd4e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-03T08:59:26.638 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:59:26 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-2[40608]: 2024-05-03T08:59:26.258+0000 7f1d3fd4e700 -1 osd.2 69 *** Got signal Terminated *** 2024-05-03T08:59:26.638 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:59:26 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-2[40608]: 2024-05-03T08:59:26.258+0000 7f1d3fd4e700 -1 osd.2 69 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-03T08:59:27.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 7.22 MB/sec execute 39 sec latency 18318.996 ms 2024-05-03T08:59:28.345 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 7.04 MB/sec execute 40 sec latency 19319.045 ms 2024-05-03T08:59:29.346 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 6.87 MB/sec execute 41 sec latency 20319.092 ms 2024-05-03T08:59:30.346 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 6.70 MB/sec execute 42 sec latency 21319.153 ms 2024-05-03T08:59:31.346 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 6.55 MB/sec execute 43 sec latency 22319.199 ms 2024-05-03T08:59:32.028 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:59:31 smithi032 podman[55914]: 2024-05-03 08:59:31.757249206 +0000 UTC m=+5.613147145 container died 6dfd926898e3dadc9ae5f598d7d21551fda3e4f41fb8fbbbfd7ee56de050f420 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-2, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, release=754, GIT_CLEAN=True, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.openshift.expose-services=) 2024-05-03T08:59:32.346 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 6.40 MB/sec execute 44 sec latency 23319.239 ms 2024-05-03T08:59:32.388 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:59:32 smithi032 podman[55914]: 2024-05-03 08:59:32.176275819 +0000 UTC m=+6.032173759 container cleanup 6dfd926898e3dadc9ae5f598d7d21551fda3e4f41fb8fbbbfd7ee56de050f420 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-2, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, release=754, version=8, GIT_BRANCH=HEAD, RELEASE=HEAD, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True) 2024-05-03T08:59:32.388 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:59:32 smithi032 bash[55914]: ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-2 2024-05-03T08:59:32.698 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:59:32 smithi032 podman[55933]: 2024-05-03 08:59:32.422547565 +0000 UTC m=+0.660235726 container remove 6dfd926898e3dadc9ae5f598d7d21551fda3e4f41fb8fbbbfd7ee56de050f420 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-2, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, version=8, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, architecture=x86_64) 2024-05-03T08:59:32.699 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:59:32 smithi032 bash[55947]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd.2" found: no such container 2024-05-03T08:59:33.007 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:59:32 smithi032 podman[56009]: 2024-05-03 08:59:32.603952718 +0000 UTC m=+0.022006096 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-03T08:59:33.007 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:59:32 smithi032 podman[56009]: 2024-05-03 08:59:32.745607587 +0000 UTC m=+0.163660948 container create 289d7ad5a97175253bd1b79e87eb761808711c09cc843f31da44371b1ed7a953 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-2-deactivate, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, name=centos-stream, version=8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, ceph=True, build-date=2022-05-03T08:36:31.336870) 2024-05-03T08:59:33.007 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:59:33 smithi032 podman[56009]: 2024-05-03 08:59:33.006380615 +0000 UTC m=+0.424433976 container init 289d7ad5a97175253bd1b79e87eb761808711c09cc843f31da44371b1ed7a953 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-2-deactivate, io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.openshift.expose-services=, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , RELEASE=HEAD, distribution-scope=public, vendor=Red Hat, Inc.) 2024-05-03T08:59:33.346 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 6.26 MB/sec execute 45 sec latency 24319.287 ms 2024-05-03T08:59:33.376 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:59:33 smithi032 podman[56009]: 2024-05-03 08:59:33.017020404 +0000 UTC m=+0.435073766 container start 289d7ad5a97175253bd1b79e87eb761808711c09cc843f31da44371b1ed7a953 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-2-deactivate, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, version=8, com.redhat.component=centos-stream-container, GIT_CLEAN=True, release=754, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, distribution-scope=public, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc.) 2024-05-03T08:59:33.376 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:59:33 smithi032 podman[56009]: 2024-05-03 08:59:33.123080214 +0000 UTC m=+0.541133584 container attach 289d7ad5a97175253bd1b79e87eb761808711c09cc843f31da44371b1ed7a953 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-2-deactivate, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , version=8, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vcs-type=git, GIT_CLEAN=True, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, ceph=True) 2024-05-03T08:59:33.638 INFO:journalctl@ceph.osd.2.smithi032.stdout:May 03 08:59:33 smithi032 podman[56009]: 2024-05-03 08:59:33.375617677 +0000 UTC m=+0.793671045 container died 289d7ad5a97175253bd1b79e87eb761808711c09cc843f31da44371b1ed7a953 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-2-deactivate, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , distribution-scope=public, name=centos-stream, architecture=x86_64, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream) 2024-05-03T08:59:34.087 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.2.service' 2024-05-03T08:59:34.123 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-03T08:59:34.123 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-05-03T08:59:34.124 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-05-03T08:59:34.124 DEBUG:teuthology.orchestra.run.smithi032:> sudo systemctl stop ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.3 2024-05-03T08:59:34.346 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 6.12 MB/sec execute 46 sec latency 25319.567 ms 2024-05-03T08:59:34.386 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:59:34 smithi032 systemd[1]: Stopping Ceph osd.3 for df63fec2-0929-11ef-bc97-c7b262605968... 2024-05-03T08:59:34.386 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:59:34 smithi032 bash[56088]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd.3" found: no such container 2024-05-03T08:59:34.638 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:59:34 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-3[43892]: 2024-05-03T08:59:34.384+0000 7f511b264700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-03T08:59:34.638 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:59:34 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-3[43892]: 2024-05-03T08:59:34.384+0000 7f511b264700 -1 osd.3 69 *** Got signal Terminated *** 2024-05-03T08:59:34.638 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:59:34 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-3[43892]: 2024-05-03T08:59:34.384+0000 7f511b264700 -1 osd.3 69 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-03T08:59:35.346 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 5.99 MB/sec execute 47 sec latency 26319.615 ms 2024-05-03T08:59:36.346 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 5.87 MB/sec execute 48 sec latency 27319.663 ms 2024-05-03T08:59:37.346 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 5.75 MB/sec execute 49 sec latency 28319.725 ms 2024-05-03T08:59:37.388 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:59:37 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-3[43892]: 2024-05-03T08:59:37.036+0000 7f5118c01700 -1 osd.3 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:13.633948+0000 front 2024-05-03T08:59:13.633878+0000 (oldest deadline 2024-05-03T08:59:37.034346+0000) 2024-05-03T08:59:37.477 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:37 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4[28726]: 2024-05-03T08:59:37.050+0000 7f3106fc7700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:11.078232+0000 front 2024-05-03T08:59:11.078153+0000 (oldest deadline 2024-05-03T08:59:36.978249+0000) 2024-05-03T08:59:38.346 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 5.63 MB/sec execute 50 sec latency 29319.771 ms 2024-05-03T08:59:38.388 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:59:38 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-3[43892]: 2024-05-03T08:59:38.066+0000 7f5118c01700 -1 osd.3 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:13.633948+0000 front 2024-05-03T08:59:13.633878+0000 (oldest deadline 2024-05-03T08:59:37.034346+0000) 2024-05-03T08:59:38.477 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:38 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4[28726]: 2024-05-03T08:59:38.073+0000 7f3106fc7700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:11.078232+0000 front 2024-05-03T08:59:11.078153+0000 (oldest deadline 2024-05-03T08:59:36.978249+0000) 2024-05-03T08:59:39.227 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:39 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4[28726]: 2024-05-03T08:59:39.076+0000 7f3106fc7700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:11.078232+0000 front 2024-05-03T08:59:11.078153+0000 (oldest deadline 2024-05-03T08:59:36.978249+0000) 2024-05-03T08:59:39.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:38 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:38.942+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:39.346 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 5.52 MB/sec execute 51 sec latency 30319.820 ms 2024-05-03T08:59:39.388 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:59:39 smithi032 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-3[43892]: 2024-05-03T08:59:39.102+0000 7f5118c01700 -1 osd.3 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:13.633948+0000 front 2024-05-03T08:59:13.633878+0000 (oldest deadline 2024-05-03T08:59:37.034346+0000) 2024-05-03T08:59:39.922 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:39 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:39.521+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:15.058898+0000 front 2024-05-03T08:59:15.058924+0000 (oldest deadline 2024-05-03T08:59:38.558839+0000) 2024-05-03T08:59:40.131 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:59:39 smithi032 podman[56101]: 2024-05-03 08:59:39.864025142 +0000 UTC m=+5.590767559 container died 947099fbe82642ba8c4355095398dc8989ca79d56493d9cca310699896234528 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-3, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., name=centos-stream, GIT_CLEAN=True, vcs-type=git, architecture=x86_64, distribution-scope=public, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-05-03T08:59:40.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:39 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:39.921+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:40.227 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:40 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4[28726]: 2024-05-03T08:59:40.070+0000 7f3106fc7700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:11.078232+0000 front 2024-05-03T08:59:11.078153+0000 (oldest deadline 2024-05-03T08:59:36.978249+0000) 2024-05-03T08:59:40.346 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 5.41 MB/sec execute 52 sec latency 31319.845 ms 2024-05-03T08:59:40.388 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:59:40 smithi032 podman[56101]: 2024-05-03 08:59:40.204232705 +0000 UTC m=+5.930975104 container cleanup 947099fbe82642ba8c4355095398dc8989ca79d56493d9cca310699896234528 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-3, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, distribution-scope=public, release=754, version=8, CEPH_POINT_RELEASE=-17.2.0, ceph=True, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, RELEASE=HEAD, vendor=Red Hat, Inc., io.openshift.expose-services=, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, vcs-type=git) 2024-05-03T08:59:40.388 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:59:40 smithi032 bash[56101]: ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-3 2024-05-03T08:59:40.664 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:59:40 smithi032 podman[56119]: 2024-05-03 08:59:40.388812338 +0000 UTC m=+0.517654719 container remove 947099fbe82642ba8c4355095398dc8989ca79d56493d9cca310699896234528 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-3, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, release=754, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, name=centos-stream, io.buildah.version=1.19.8, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, version=8, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, RELEASE=HEAD) 2024-05-03T08:59:40.665 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:59:40 smithi032 bash[56131]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd.3" found: no such container 2024-05-03T08:59:40.955 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:40 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:40.550+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:15.058898+0000 front 2024-05-03T08:59:15.058924+0000 (oldest deadline 2024-05-03T08:59:38.558839+0000) 2024-05-03T08:59:40.956 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:40 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:40.640+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T08:59:40.959 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:59:40 smithi032 podman[56190]: 2024-05-03 08:59:40.569941913 +0000 UTC m=+0.022431951 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-03T08:59:40.959 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:59:40 smithi032 podman[56190]: 2024-05-03 08:59:40.723117923 +0000 UTC m=+0.175607937 container create 759b4a82e7b8a43f1c4d604b97eb94cde693006ada4b9a689f0bab7d0eb08a49 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-3-deactivate, version=8, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-03T08:59:40.960 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:59:40 smithi032 podman[56190]: 2024-05-03 08:59:40.958683302 +0000 UTC m=+0.411173318 container init 759b4a82e7b8a43f1c4d604b97eb94cde693006ada4b9a689f0bab7d0eb08a49 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-3-deactivate, release=754, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, vcs-type=git, ceph=True, distribution-scope=public, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, version=8, architecture=x86_64, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD) 2024-05-03T08:59:41.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:40 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:40.953+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:41.227 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:41 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4[28726]: 2024-05-03T08:59:41.104+0000 7f3106fc7700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:11.078232+0000 front 2024-05-03T08:59:11.078153+0000 (oldest deadline 2024-05-03T08:59:36.978249+0000) 2024-05-03T08:59:41.322 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:59:40 smithi032 podman[56190]: 2024-05-03 08:59:40.962606193 +0000 UTC m=+0.415096218 container start 759b4a82e7b8a43f1c4d604b97eb94cde693006ada4b9a689f0bab7d0eb08a49 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-3-deactivate, ceph=True, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, release=754, GIT_CLEAN=True, io.openshift.expose-services=, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-05-03T08:59:41.322 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:59:41 smithi032 podman[56190]: 2024-05-03 08:59:41.050404369 +0000 UTC m=+0.502894399 container attach 759b4a82e7b8a43f1c4d604b97eb94cde693006ada4b9a689f0bab7d0eb08a49 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-3-deactivate, vcs-type=git, name=centos-stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , version=8, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, architecture=x86_64, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-05-03T08:59:41.346 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 5.31 MB/sec execute 53 sec latency 32319.888 ms 2024-05-03T08:59:41.638 INFO:journalctl@ceph.osd.3.smithi032.stdout:May 03 08:59:41 smithi032 podman[56190]: 2024-05-03 08:59:41.321077433 +0000 UTC m=+0.773567450 container died 759b4a82e7b8a43f1c4d604b97eb94cde693006ada4b9a689f0bab7d0eb08a49 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-3-deactivate, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, release=754, build-date=2022-05-03T08:36:31.336870, version=8, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, GIT_CLEAN=True, vcs-type=git, maintainer=Guillaume Abrioux , architecture=x86_64, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_BRANCH=HEAD, ceph=True, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container) 2024-05-03T08:59:41.977 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:41 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:41.626+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T08:59:41.977 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:41 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:41.547+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:15.058898+0000 front 2024-05-03T08:59:15.058924+0000 (oldest deadline 2024-05-03T08:59:38.558839+0000) 2024-05-03T08:59:41.984 DEBUG:teuthology.orchestra.run.smithi032:> sudo pkill -f 'journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.3.service' 2024-05-03T08:59:42.020 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-03T08:59:42.021 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-05-03T08:59:42.021 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-05-03T08:59:42.021 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.4 2024-05-03T08:59:42.346 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 5.21 MB/sec execute 54 sec latency 33319.931 ms 2024-05-03T08:59:42.384 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:41 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:41.998+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:42.385 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:42 smithi123 systemd[1]: Stopping Ceph osd.4 for df63fec2-0929-11ef-bc97-c7b262605968... 2024-05-03T08:59:42.385 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:42 smithi123 bash[40715]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd.4" found: no such container 2024-05-03T08:59:42.385 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:42 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4[28726]: 2024-05-03T08:59:42.110+0000 7f3106fc7700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:11.078232+0000 front 2024-05-03T08:59:11.078153+0000 (oldest deadline 2024-05-03T08:59:36.978249+0000) 2024-05-03T08:59:42.385 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:42 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4[28726]: 2024-05-03T08:59:42.383+0000 7f3109e2b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-03T08:59:42.727 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:42 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:42.621+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T08:59:42.727 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:42 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4[28726]: 2024-05-03T08:59:42.383+0000 7f3109e2b700 -1 osd.4 69 *** Got signal Terminated *** 2024-05-03T08:59:42.727 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:42 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4[28726]: 2024-05-03T08:59:42.383+0000 7f3109e2b700 -1 osd.4 69 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-03T08:59:42.727 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:42 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:42.507+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:15.058898+0000 front 2024-05-03T08:59:15.058924+0000 (oldest deadline 2024-05-03T08:59:38.558839+0000) 2024-05-03T08:59:43.230 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:42 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:42.963+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:43.230 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:43 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4[28726]: 2024-05-03T08:59:43.136+0000 7f3106fc7700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:11.078232+0000 front 2024-05-03T08:59:11.078153+0000 (oldest deadline 2024-05-03T08:59:36.978249+0000) 2024-05-03T08:59:43.346 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 5.12 MB/sec execute 55 sec latency 34319.975 ms 2024-05-03T08:59:43.941 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:43 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:43.523+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:15.058898+0000 front 2024-05-03T08:59:15.058924+0000 (oldest deadline 2024-05-03T08:59:38.558839+0000) 2024-05-03T08:59:43.942 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:43 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:43.610+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T08:59:44.227 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:44 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4[28726]: 2024-05-03T08:59:44.181+0000 7f3106fc7700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:11.078232+0000 front 2024-05-03T08:59:11.078153+0000 (oldest deadline 2024-05-03T08:59:36.978249+0000) 2024-05-03T08:59:44.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:43 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:43.940+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:44.346 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 5.03 MB/sec execute 56 sec latency 35320.021 ms 2024-05-03T08:59:44.928 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:44 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:44.645+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T08:59:44.929 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:44 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:44.552+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:15.058898+0000 front 2024-05-03T08:59:15.058924+0000 (oldest deadline 2024-05-03T08:59:38.558839+0000) 2024-05-03T08:59:45.227 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:45 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4[28726]: 2024-05-03T08:59:45.163+0000 7f3106fc7700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:11.078232+0000 front 2024-05-03T08:59:11.078153+0000 (oldest deadline 2024-05-03T08:59:36.978249+0000) 2024-05-03T08:59:45.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:44 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:44.927+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:45.346 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 4.94 MB/sec execute 57 sec latency 36320.047 ms 2024-05-03T08:59:45.901 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:45 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:45.607+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T08:59:45.902 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:45 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:45.508+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:15.058898+0000 front 2024-05-03T08:59:15.058924+0000 (oldest deadline 2024-05-03T08:59:38.558839+0000) 2024-05-03T08:59:45.902 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:45 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:45.508+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:22.460761+0000 front 2024-05-03T08:59:22.464950+0000 (oldest deadline 2024-05-03T08:59:45.361016+0000) 2024-05-03T08:59:46.227 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:46 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4[28726]: 2024-05-03T08:59:46.118+0000 7f3106fc7700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:11.078232+0000 front 2024-05-03T08:59:11.078153+0000 (oldest deadline 2024-05-03T08:59:36.978249+0000) 2024-05-03T08:59:46.228 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:45 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:45.899+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:46.346 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 4.85 MB/sec execute 58 sec latency 37320.094 ms 2024-05-03T08:59:46.933 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:46 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:46.524+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:15.058898+0000 front 2024-05-03T08:59:15.058924+0000 (oldest deadline 2024-05-03T08:59:38.558839+0000) 2024-05-03T08:59:46.933 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:46 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:46.524+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:22.460761+0000 front 2024-05-03T08:59:22.464950+0000 (oldest deadline 2024-05-03T08:59:45.361016+0000) 2024-05-03T08:59:46.933 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:46 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:46.642+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T08:59:46.933 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:46 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:46.642+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T08:59:47.227 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:47 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4[28726]: 2024-05-03T08:59:47.080+0000 7f3106fc7700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:11.078232+0000 front 2024-05-03T08:59:11.078153+0000 (oldest deadline 2024-05-03T08:59:36.978249+0000) 2024-05-03T08:59:47.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:46 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:46.931+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:47.346 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 4.77 MB/sec execute 59 sec latency 38320.151 ms 2024-05-03T08:59:47.948 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:47 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:47.544+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:15.058898+0000 front 2024-05-03T08:59:15.058924+0000 (oldest deadline 2024-05-03T08:59:38.558839+0000) 2024-05-03T08:59:47.948 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:47 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:47.544+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:22.460761+0000 front 2024-05-03T08:59:22.464950+0000 (oldest deadline 2024-05-03T08:59:45.361016+0000) 2024-05-03T08:59:47.948 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:47 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:47.608+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T08:59:47.948 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:47 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:47.608+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T08:59:48.227 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:47 smithi123 podman[40728]: 2024-05-03 08:59:47.946865071 +0000 UTC m=+5.842026884 container died 150ddc5685f9e4907eeba898e42eb81a8440fae13f7763c6e7065413cbdeb557 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4, architecture=x86_64, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, vcs-type=git, release=754, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870) 2024-05-03T08:59:48.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:47 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:47.958+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:48.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:47 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:47.958+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:18.358517+0000 front 2024-05-03T08:59:18.358599+0000 (oldest deadline 2024-05-03T08:59:47.759248+0000) 2024-05-03T08:59:48.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 4.69 MB/sec execute 60 sec latency 39320.197 ms 2024-05-03T08:59:48.569 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:48 smithi123 podman[40728]: 2024-05-03 08:59:48.416549697 +0000 UTC m=+6.311711509 container cleanup 150ddc5685f9e4907eeba898e42eb81a8440fae13f7763c6e7065413cbdeb557 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, RELEASE=HEAD, GIT_CLEAN=True, name=centos-stream, version=8, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, release=754, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, GIT_BRANCH=HEAD, distribution-scope=public, architecture=x86_64, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-05-03T08:59:48.569 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:48 smithi123 bash[40728]: ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4 2024-05-03T08:59:48.570 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:48 smithi123 bash[40757]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd.4" found: no such container 2024-05-03T08:59:48.852 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:48 smithi123 podman[40745]: 2024-05-03 08:59:48.659574678 +0000 UTC m=+0.713661677 container remove 150ddc5685f9e4907eeba898e42eb81a8440fae13f7763c6e7065413cbdeb557 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, version=8, vcs-type=git, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, RELEASE=HEAD, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0) 2024-05-03T08:59:48.853 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:48 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:48.568+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:15.058898+0000 front 2024-05-03T08:59:15.058924+0000 (oldest deadline 2024-05-03T08:59:38.558839+0000) 2024-05-03T08:59:48.853 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:48 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:48.568+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:22.460761+0000 front 2024-05-03T08:59:22.464950+0000 (oldest deadline 2024-05-03T08:59:45.361016+0000) 2024-05-03T08:59:48.853 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:48 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:48.619+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T08:59:48.854 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:48 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:48.619+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T08:59:49.164 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:48 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:48.969+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:49.164 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:48 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:48.969+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:18.358517+0000 front 2024-05-03T08:59:18.358599+0000 (oldest deadline 2024-05-03T08:59:47.759248+0000) 2024-05-03T08:59:49.164 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:48 smithi123 podman[40819]: 2024-05-03 08:59:48.755308419 +0000 UTC m=+0.019523685 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-03T08:59:49.164 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:48 smithi123 podman[40819]: 2024-05-03 08:59:48.852143058 +0000 UTC m=+0.116358340 container create 370147a07f9d8c51b276a5fad97c193eec7e96eeb7d390437dc49ba9555adff5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4-deactivate, version=8, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, architecture=x86_64, vcs-type=git, ceph=True, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-05-03T08:59:49.164 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:49 smithi123 podman[40819]: 2024-05-03 08:59:49.163043709 +0000 UTC m=+0.427258946 container init 370147a07f9d8c51b276a5fad97c193eec7e96eeb7d390437dc49ba9555adff5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4-deactivate, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, GIT_BRANCH=HEAD, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, ceph=True, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, architecture=x86_64, name=centos-stream, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, io.buildah.version=1.19.8) 2024-05-03T08:59:49.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 4.62 MB/sec execute 61 sec latency 40320.237 ms 2024-05-03T08:59:49.446 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:49 smithi123 podman[40819]: 2024-05-03 08:59:49.166241728 +0000 UTC m=+0.430456971 container start 370147a07f9d8c51b276a5fad97c193eec7e96eeb7d390437dc49ba9555adff5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4-deactivate, architecture=x86_64, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, ceph=True, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, version=8, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0) 2024-05-03T08:59:49.447 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:49 smithi123 podman[40819]: 2024-05-03 08:59:49.254662774 +0000 UTC m=+0.518878019 container attach 370147a07f9d8c51b276a5fad97c193eec7e96eeb7d390437dc49ba9555adff5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4-deactivate, io.buildah.version=1.19.8, name=centos-stream, distribution-scope=public, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, architecture=x86_64, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.openshift.expose-services=, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD) 2024-05-03T08:59:49.727 INFO:journalctl@ceph.osd.4.smithi123.stdout:May 03 08:59:49 smithi123 podman[40819]: 2024-05-03 08:59:49.445821013 +0000 UTC m=+0.710036255 container died 370147a07f9d8c51b276a5fad97c193eec7e96eeb7d390437dc49ba9555adff5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-4-deactivate, vcs-type=git, ceph=True, io.openshift.tags=base centos centos-stream, architecture=x86_64, GIT_CLEAN=True, name=centos-stream, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, release=754, CEPH_POINT_RELEASE=-17.2.0) 2024-05-03T08:59:49.727 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:49 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:49.554+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:15.058898+0000 front 2024-05-03T08:59:15.058924+0000 (oldest deadline 2024-05-03T08:59:38.558839+0000) 2024-05-03T08:59:49.727 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:49 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:49.554+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:22.460761+0000 front 2024-05-03T08:59:22.464950+0000 (oldest deadline 2024-05-03T08:59:45.361016+0000) 2024-05-03T08:59:49.728 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:49 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:49.614+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T08:59:49.728 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:49 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:49.614+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T08:59:50.050 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:49 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:49.994+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:50.051 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:49 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:49.994+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:18.358517+0000 front 2024-05-03T08:59:18.358599+0000 (oldest deadline 2024-05-03T08:59:47.759248+0000) 2024-05-03T08:59:50.069 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.4.service' 2024-05-03T08:59:50.107 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-03T08:59:50.108 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-05-03T08:59:50.108 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-05-03T08:59:50.108 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.5 2024-05-03T08:59:50.320 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:50 smithi123 systemd[1]: Stopping Ceph osd.5 for df63fec2-0929-11ef-bc97-c7b262605968... 2024-05-03T08:59:50.321 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:50 smithi123 bash[40899]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd.5" found: no such container 2024-05-03T08:59:50.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 4.54 MB/sec execute 62 sec latency 41320.281 ms 2024-05-03T08:59:50.664 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:50 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:50.318+0000 7f3c592b4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-03T08:59:50.664 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:50 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:50.318+0000 7f3c592b4700 -1 osd.5 69 *** Got signal Terminated *** 2024-05-03T08:59:50.664 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:50 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:50.318+0000 7f3c592b4700 -1 osd.5 69 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-03T08:59:50.664 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:50 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:50.525+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:15.058898+0000 front 2024-05-03T08:59:15.058924+0000 (oldest deadline 2024-05-03T08:59:38.558839+0000) 2024-05-03T08:59:50.664 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:50 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:50.525+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:22.460761+0000 front 2024-05-03T08:59:22.464950+0000 (oldest deadline 2024-05-03T08:59:45.361016+0000) 2024-05-03T08:59:50.977 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:50 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:50.661+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T08:59:50.977 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:50 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:50.662+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T08:59:51.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 4.47 MB/sec execute 63 sec latency 42320.343 ms 2024-05-03T08:59:51.477 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:51 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:51.014+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:51.477 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:51 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:51.014+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:18.358517+0000 front 2024-05-03T08:59:18.358599+0000 (oldest deadline 2024-05-03T08:59:47.759248+0000) 2024-05-03T08:59:51.977 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:51 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:51.567+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:15.058898+0000 front 2024-05-03T08:59:15.058924+0000 (oldest deadline 2024-05-03T08:59:38.558839+0000) 2024-05-03T08:59:51.977 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:51 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:51.567+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:22.460761+0000 front 2024-05-03T08:59:22.464950+0000 (oldest deadline 2024-05-03T08:59:45.361016+0000) 2024-05-03T08:59:51.977 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:51 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:51.670+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T08:59:51.978 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:51 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:51.670+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T08:59:52.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 4.40 MB/sec execute 64 sec latency 43320.392 ms 2024-05-03T08:59:52.477 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:52 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:52.064+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:52.477 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:52 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:52.064+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:18.358517+0000 front 2024-05-03T08:59:18.358599+0000 (oldest deadline 2024-05-03T08:59:47.759248+0000) 2024-05-03T08:59:52.477 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:52 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:52.064+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:27.759705+0000 front 2024-05-03T08:59:27.759748+0000 (oldest deadline 2024-05-03T08:59:51.760190+0000) 2024-05-03T08:59:52.977 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:52 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:52.609+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:15.058898+0000 front 2024-05-03T08:59:15.058924+0000 (oldest deadline 2024-05-03T08:59:38.558839+0000) 2024-05-03T08:59:52.977 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:52 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:52.609+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:22.460761+0000 front 2024-05-03T08:59:22.464950+0000 (oldest deadline 2024-05-03T08:59:45.361016+0000) 2024-05-03T08:59:52.977 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:52 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:52.714+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T08:59:52.978 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:52 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:52.714+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T08:59:52.978 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:52 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:52.714+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:26.437726+0000 front 2024-05-03T08:59:26.438420+0000 (oldest deadline 2024-05-03T08:59:52.337769+0000) 2024-05-03T08:59:53.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 4.33 MB/sec execute 65 sec latency 44320.433 ms 2024-05-03T08:59:53.477 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:53 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:53.050+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:53.477 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:53 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:53.050+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:18.358517+0000 front 2024-05-03T08:59:18.358599+0000 (oldest deadline 2024-05-03T08:59:47.759248+0000) 2024-05-03T08:59:53.477 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:53 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:53.050+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:27.759705+0000 front 2024-05-03T08:59:27.759748+0000 (oldest deadline 2024-05-03T08:59:51.760190+0000) 2024-05-03T08:59:53.977 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:53 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:53.599+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:15.058898+0000 front 2024-05-03T08:59:15.058924+0000 (oldest deadline 2024-05-03T08:59:38.558839+0000) 2024-05-03T08:59:53.977 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:53 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:53.599+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:22.460761+0000 front 2024-05-03T08:59:22.464950+0000 (oldest deadline 2024-05-03T08:59:45.361016+0000) 2024-05-03T08:59:53.978 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:53 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:53.753+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T08:59:53.978 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:53 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:53.753+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T08:59:53.978 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:53 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:53.753+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:26.437726+0000 front 2024-05-03T08:59:26.438420+0000 (oldest deadline 2024-05-03T08:59:52.337769+0000) 2024-05-03T08:59:54.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 4.27 MB/sec execute 66 sec latency 45320.481 ms 2024-05-03T08:59:54.477 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:54 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:54.078+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:54.477 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:54 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:54.078+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:18.358517+0000 front 2024-05-03T08:59:18.358599+0000 (oldest deadline 2024-05-03T08:59:47.759248+0000) 2024-05-03T08:59:54.477 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:54 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:54.078+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:27.759705+0000 front 2024-05-03T08:59:27.759748+0000 (oldest deadline 2024-05-03T08:59:51.760190+0000) 2024-05-03T08:59:54.977 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:54 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:54.631+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:15.058898+0000 front 2024-05-03T08:59:15.058924+0000 (oldest deadline 2024-05-03T08:59:38.558839+0000) 2024-05-03T08:59:54.977 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:54 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5[31690]: 2024-05-03T08:59:54.631+0000 7f3c5544e700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:22.460761+0000 front 2024-05-03T08:59:22.464950+0000 (oldest deadline 2024-05-03T08:59:45.361016+0000) 2024-05-03T08:59:54.978 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:54 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:54.715+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T08:59:54.978 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:54 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:54.715+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T08:59:54.978 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:54 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:54.715+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:26.437726+0000 front 2024-05-03T08:59:26.438420+0000 (oldest deadline 2024-05-03T08:59:52.337769+0000) 2024-05-03T08:59:55.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 4.20 MB/sec execute 67 sec latency 46320.530 ms 2024-05-03T08:59:55.477 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:55 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:55.083+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:55.477 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:55 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:55.083+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:18.358517+0000 front 2024-05-03T08:59:18.358599+0000 (oldest deadline 2024-05-03T08:59:47.759248+0000) 2024-05-03T08:59:55.477 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:55 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:55.083+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:27.759705+0000 front 2024-05-03T08:59:27.759748+0000 (oldest deadline 2024-05-03T08:59:51.760190+0000) 2024-05-03T08:59:55.977 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:55 smithi123 podman[40910]: 2024-05-03 08:59:55.742641707 +0000 UTC m=+5.529625865 container died 72ba92ce4b536a6078d8d6458eeae59f44fb0b5158e14c14ad13d23e952c4bdc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5, io.openshift.expose-services=, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, version=8, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, ceph=True, distribution-scope=public, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, name=centos-stream, GIT_BRANCH=HEAD) 2024-05-03T08:59:55.977 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:55 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:55.696+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T08:59:55.977 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:55 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:55.696+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T08:59:55.978 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:55 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:55.696+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:26.437726+0000 front 2024-05-03T08:59:26.438420+0000 (oldest deadline 2024-05-03T08:59:52.337769+0000) 2024-05-03T08:59:56.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 4.14 MB/sec execute 68 sec latency 47320.569 ms 2024-05-03T08:59:56.413 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:56 smithi123 podman[40910]: 2024-05-03 08:59:56.12710074 +0000 UTC m=+5.914084896 container cleanup 72ba92ce4b536a6078d8d6458eeae59f44fb0b5158e14c14ad13d23e952c4bdc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5, build-date=2022-05-03T08:36:31.336870, vcs-type=git, version=8, architecture=x86_64, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, io.buildah.version=1.19.8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.component=centos-stream-container, release=754, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.openshift.expose-services=) 2024-05-03T08:59:56.413 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:56 smithi123 bash[40910]: ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5 2024-05-03T08:59:56.413 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:56 smithi123 bash[40954]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd.5" found: no such container 2024-05-03T08:59:56.413 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:56 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:56.038+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:56.413 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:56 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:56.038+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:18.358517+0000 front 2024-05-03T08:59:18.358599+0000 (oldest deadline 2024-05-03T08:59:47.759248+0000) 2024-05-03T08:59:56.414 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:56 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:56.038+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:27.759705+0000 front 2024-05-03T08:59:27.759748+0000 (oldest deadline 2024-05-03T08:59:51.760190+0000) 2024-05-03T08:59:56.675 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:56 smithi123 podman[40939]: 2024-05-03 08:59:56.411706759 +0000 UTC m=+0.662122156 container remove 72ba92ce4b536a6078d8d6458eeae59f44fb0b5158e14c14ad13d23e952c4bdc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.19.8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, architecture=x86_64, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-05-03T08:59:56.675 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:56 smithi123 podman[41011]: 2024-05-03 08:59:56.468754455 +0000 UTC m=+0.018719508 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-03T08:59:56.675 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:56 smithi123 podman[41011]: 2024-05-03 08:59:56.579296014 +0000 UTC m=+0.129261056 container create 09354937d9554816eeb974ccc08034edbc9f6f3dc85e812d37f93b0d1e50e41c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, vcs-type=git, ceph=True, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, maintainer=Guillaume Abrioux , RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, release=754, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public) 2024-05-03T08:59:57.061 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:56 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:56.743+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T08:59:57.061 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:56 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:56.743+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T08:59:57.061 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:56 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:56.743+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:26.437726+0000 front 2024-05-03T08:59:26.438420+0000 (oldest deadline 2024-05-03T08:59:52.337769+0000) 2024-05-03T08:59:57.062 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:56 smithi123 podman[41011]: 2024-05-03 08:59:56.79003596 +0000 UTC m=+0.340001003 container init 09354937d9554816eeb974ccc08034edbc9f6f3dc85e812d37f93b0d1e50e41c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5-deactivate, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, architecture=x86_64, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, GIT_BRANCH=HEAD, release=754) 2024-05-03T08:59:57.062 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:56 smithi123 podman[41011]: 2024-05-03 08:59:56.793182813 +0000 UTC m=+0.343147862 container start 09354937d9554816eeb974ccc08034edbc9f6f3dc85e812d37f93b0d1e50e41c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5-deactivate, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, ceph=True, architecture=x86_64, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vendor=Red Hat, Inc., GIT_CLEAN=True, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, version=8, distribution-scope=public, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container) 2024-05-03T08:59:57.062 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:56 smithi123 podman[41011]: 2024-05-03 08:59:56.87303937 +0000 UTC m=+0.423004414 container attach 09354937d9554816eeb974ccc08034edbc9f6f3dc85e812d37f93b0d1e50e41c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5-deactivate, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, release=754, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, architecture=x86_64, GIT_CLEAN=True, vcs-type=git, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8) 2024-05-03T08:59:57.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 4.08 MB/sec execute 69 sec latency 48320.628 ms 2024-05-03T08:59:57.418 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:57 smithi123 podman[41011]: 2024-05-03 08:59:57.071530745 +0000 UTC m=+0.621495789 container died 09354937d9554816eeb974ccc08034edbc9f6f3dc85e812d37f93b0d1e50e41c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5-deactivate, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, distribution-scope=public, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, name=centos-stream, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.openshift.expose-services=, GIT_CLEAN=True, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-05-03T08:59:57.419 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:57 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:57.059+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:57.419 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:57 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:57.059+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:18.358517+0000 front 2024-05-03T08:59:18.358599+0000 (oldest deadline 2024-05-03T08:59:47.759248+0000) 2024-05-03T08:59:57.419 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:57 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:57.059+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:27.759705+0000 front 2024-05-03T08:59:27.759748+0000 (oldest deadline 2024-05-03T08:59:51.760190+0000) 2024-05-03T08:59:57.679 INFO:journalctl@ceph.osd.5.smithi123.stdout:May 03 08:59:57 smithi123 podman[41037]: 2024-05-03 08:59:57.659568223 +0000 UTC m=+0.581558393 container remove 09354937d9554816eeb974ccc08034edbc9f6f3dc85e812d37f93b0d1e50e41c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-5-deactivate, vendor=Red Hat, Inc., version=8, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, release=754, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-05-03T08:59:57.684 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.5.service' 2024-05-03T08:59:57.722 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-03T08:59:57.723 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-05-03T08:59:57.723 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-05-03T08:59:57.723 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.6 2024-05-03T08:59:57.947 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:57 smithi123 systemd[1]: Stopping Ceph osd.6 for df63fec2-0929-11ef-bc97-c7b262605968... 2024-05-03T08:59:57.947 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:57 smithi123 bash[41090]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd.6" found: no such container 2024-05-03T08:59:57.947 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:57 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:57.945+0000 7f7f2c9ab700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-03T08:59:57.947 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:57 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:57.945+0000 7f7f2c9ab700 -1 osd.6 69 *** Got signal Terminated *** 2024-05-03T08:59:57.947 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:57 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:57.945+0000 7f7f2c9ab700 -1 osd.6 69 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-03T08:59:57.948 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:57 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:57.772+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T08:59:57.948 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:57 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:57.772+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T08:59:57.948 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:57 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:57.772+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:26.437726+0000 front 2024-05-03T08:59:26.438420+0000 (oldest deadline 2024-05-03T08:59:52.337769+0000) 2024-05-03T08:59:58.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:58 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:58.083+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:58.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:58 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:58.083+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:18.358517+0000 front 2024-05-03T08:59:18.358599+0000 (oldest deadline 2024-05-03T08:59:47.759248+0000) 2024-05-03T08:59:58.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:58 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:58.083+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:27.759705+0000 front 2024-05-03T08:59:27.759748+0000 (oldest deadline 2024-05-03T08:59:51.760190+0000) 2024-05-03T08:59:58.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 4.02 MB/sec execute 70 sec latency 49320.669 ms 2024-05-03T08:59:59.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:59 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:59.051+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T08:59:59.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:59 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:59.051+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:18.358517+0000 front 2024-05-03T08:59:18.358599+0000 (oldest deadline 2024-05-03T08:59:47.759248+0000) 2024-05-03T08:59:59.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 08:59:59 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T08:59:59.051+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:27.759705+0000 front 2024-05-03T08:59:27.759748+0000 (oldest deadline 2024-05-03T08:59:51.760190+0000) 2024-05-03T08:59:59.227 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:58 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:58.810+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T08:59:59.227 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:58 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:58.810+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T08:59:59.228 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:58 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:58.810+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:26.437726+0000 front 2024-05-03T08:59:26.438420+0000 (oldest deadline 2024-05-03T08:59:52.337769+0000) 2024-05-03T08:59:59.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.97 MB/sec execute 71 sec latency 50320.724 ms 2024-05-03T09:00:00.070 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:59 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:59.773+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T09:00:00.071 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:59 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:59.773+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T09:00:00.071 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 08:59:59 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T08:59:59.773+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:26.437726+0000 front 2024-05-03T08:59:26.438420+0000 (oldest deadline 2024-05-03T08:59:52.337769+0000) 2024-05-03T09:00:00.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.91 MB/sec execute 72 sec latency 51320.748 ms 2024-05-03T09:00:00.477 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:00 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T09:00:00.068+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T09:00:00.477 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:00 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T09:00:00.068+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:18.358517+0000 front 2024-05-03T08:59:18.358599+0000 (oldest deadline 2024-05-03T08:59:47.759248+0000) 2024-05-03T09:00:00.477 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:00 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T09:00:00.068+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:27.759705+0000 front 2024-05-03T08:59:27.759748+0000 (oldest deadline 2024-05-03T08:59:51.760190+0000) 2024-05-03T09:00:01.023 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:00 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:00.753+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T09:00:01.023 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:00 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:00.753+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T09:00:01.023 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:00 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:00.753+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:26.437726+0000 front 2024-05-03T08:59:26.438420+0000 (oldest deadline 2024-05-03T08:59:52.337769+0000) 2024-05-03T09:00:01.281 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:01 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T09:00:01.021+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T09:00:01.282 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:01 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T09:00:01.021+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:18.358517+0000 front 2024-05-03T08:59:18.358599+0000 (oldest deadline 2024-05-03T08:59:47.759248+0000) 2024-05-03T09:00:01.282 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:01 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T09:00:01.021+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:27.759705+0000 front 2024-05-03T08:59:27.759748+0000 (oldest deadline 2024-05-03T08:59:51.760190+0000) 2024-05-03T09:00:01.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.86 MB/sec execute 73 sec latency 52320.800 ms 2024-05-03T09:00:02.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:01 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T09:00:01.978+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T09:00:02.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:01 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T09:00:01.978+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:18.358517+0000 front 2024-05-03T08:59:18.358599+0000 (oldest deadline 2024-05-03T08:59:47.759248+0000) 2024-05-03T09:00:02.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:01 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T09:00:01.978+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:27.759705+0000 front 2024-05-03T08:59:27.759748+0000 (oldest deadline 2024-05-03T08:59:51.760190+0000) 2024-05-03T09:00:02.228 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:01 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:01.788+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T09:00:02.228 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:01 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:01.788+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T09:00:02.228 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:01 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:01.788+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:26.437726+0000 front 2024-05-03T08:59:26.438420+0000 (oldest deadline 2024-05-03T08:59:52.337769+0000) 2024-05-03T09:00:02.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.80 MB/sec execute 74 sec latency 53320.855 ms 2024-05-03T09:00:03.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:02 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T09:00:02.937+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.857845+0000 front 2024-05-03T08:59:14.857914+0000 (oldest deadline 2024-05-03T08:59:38.358061+0000) 2024-05-03T09:00:03.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:02 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T09:00:02.937+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:18.358517+0000 front 2024-05-03T08:59:18.358599+0000 (oldest deadline 2024-05-03T08:59:47.759248+0000) 2024-05-03T09:00:03.227 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:02 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6[34676]: 2024-05-03T09:00:02.937+0000 7f7f25b3f700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:27.759705+0000 front 2024-05-03T08:59:27.759748+0000 (oldest deadline 2024-05-03T08:59:51.760190+0000) 2024-05-03T09:00:03.227 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:02 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:02.824+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T09:00:03.228 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:02 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:02.824+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T09:00:03.228 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:02 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:02.824+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:26.437726+0000 front 2024-05-03T08:59:26.438420+0000 (oldest deadline 2024-05-03T08:59:52.337769+0000) 2024-05-03T09:00:03.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.75 MB/sec execute 75 sec latency 54320.900 ms 2024-05-03T09:00:03.788 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:03 smithi123 podman[41102]: 2024-05-03 09:00:03.509551728 +0000 UTC m=+5.676966550 container died 2bc752f6bfc53b918c364c6a499515271f83351b27342b1f599b007ab6512dc4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, architecture=x86_64, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , release=754, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, ceph=True, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-03T09:00:03.789 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:03 smithi123 podman[41102]: 2024-05-03 09:00:03.787081396 +0000 UTC m=+5.954496196 container cleanup 2bc752f6bfc53b918c364c6a499515271f83351b27342b1f599b007ab6512dc4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, name=centos-stream, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, release=754, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, version=8, io.buildah.version=1.19.8, ceph=True, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD) 2024-05-03T09:00:04.064 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:03 smithi123 bash[41102]: ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6 2024-05-03T09:00:04.064 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:03 smithi123 bash[41134]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd.6" found: no such container 2024-05-03T09:00:04.064 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:03 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:03.808+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T09:00:04.065 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:03 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:03.808+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T09:00:04.065 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:03 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:03.808+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:26.437726+0000 front 2024-05-03T08:59:26.438420+0000 (oldest deadline 2024-05-03T08:59:52.337769+0000) 2024-05-03T09:00:04.065 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:03 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:03.808+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6830 osd.3 since back 2024-05-03T08:59:38.739456+0000 front 2024-05-03T08:59:38.739500+0000 (oldest deadline 2024-05-03T09:00:03.439552+0000) 2024-05-03T09:00:04.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.70 MB/sec execute 76 sec latency 55320.923 ms 2024-05-03T09:00:04.413 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:04 smithi123 podman[41122]: 2024-05-03 09:00:04.06321604 +0000 UTC m=+0.546108319 container remove 2bc752f6bfc53b918c364c6a499515271f83351b27342b1f599b007ab6512dc4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, ceph=True, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, architecture=x86_64, version=8, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream) 2024-05-03T09:00:04.413 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:04 smithi123 podman[41193]: 2024-05-03 09:00:04.173733685 +0000 UTC m=+0.018869190 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-03T09:00:04.413 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:04 smithi123 podman[41193]: 2024-05-03 09:00:04.289120909 +0000 UTC m=+0.134256441 container create 7e6a982d9ca91800bb9361cc38ff994f2e4c9bbdd46b94480e404f5f37c3538f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.openshift.tags=base centos centos-stream, name=centos-stream, release=754, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, RELEASE=HEAD, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-05-03T09:00:04.727 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:04 smithi123 podman[41193]: 2024-05-03 09:00:04.541381609 +0000 UTC m=+0.386517099 container init 7e6a982d9ca91800bb9361cc38ff994f2e4c9bbdd46b94480e404f5f37c3538f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6-deactivate, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, GIT_CLEAN=True, architecture=x86_64, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , version=8, io.openshift.expose-services=, name=centos-stream) 2024-05-03T09:00:04.727 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:04 smithi123 podman[41193]: 2024-05-03 09:00:04.54448673 +0000 UTC m=+0.389622223 container start 7e6a982d9ca91800bb9361cc38ff994f2e4c9bbdd46b94480e404f5f37c3538f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6-deactivate, io.openshift.expose-services=, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, maintainer=Guillaume Abrioux , vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, ceph=True, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-05-03T09:00:04.727 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:04 smithi123 podman[41193]: 2024-05-03 09:00:04.628085945 +0000 UTC m=+0.473221453 container attach 7e6a982d9ca91800bb9361cc38ff994f2e4c9bbdd46b94480e404f5f37c3538f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6-deactivate, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, vcs-type=git, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, release=754, ceph=True, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, distribution-scope=public, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, version=8, name=centos-stream) 2024-05-03T09:00:05.087 INFO:journalctl@ceph.osd.6.smithi123.stdout:May 03 09:00:04 smithi123 podman[41193]: 2024-05-03 09:00:04.834895827 +0000 UTC m=+0.680031309 container died 7e6a982d9ca91800bb9361cc38ff994f2e4c9bbdd46b94480e404f5f37c3538f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-6-deactivate, RELEASE=HEAD, maintainer=Guillaume Abrioux , architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870) 2024-05-03T09:00:05.087 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:04 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:04.823+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T09:00:05.087 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:04 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:04.823+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T09:00:05.087 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:04 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:04.823+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:26.437726+0000 front 2024-05-03T08:59:26.438420+0000 (oldest deadline 2024-05-03T08:59:52.337769+0000) 2024-05-03T09:00:05.087 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:04 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:04.823+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6830 osd.3 since back 2024-05-03T08:59:38.739456+0000 front 2024-05-03T08:59:38.739500+0000 (oldest deadline 2024-05-03T09:00:03.439552+0000) 2024-05-03T09:00:05.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.66 MB/sec execute 77 sec latency 56320.972 ms 2024-05-03T09:00:05.395 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.6.service' 2024-05-03T09:00:05.432 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-03T09:00:05.433 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-05-03T09:00:05.433 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-05-03T09:00:05.433 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.7 2024-05-03T09:00:05.699 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:05 smithi123 systemd[1]: Stopping Ceph osd.7 for df63fec2-0929-11ef-bc97-c7b262605968... 2024-05-03T09:00:05.699 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:05 smithi123 bash[41272]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd.7" found: no such container 2024-05-03T09:00:05.699 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:05 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:05.697+0000 7fc7e2417700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-03T09:00:05.699 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:05 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:05.697+0000 7fc7e2417700 -1 osd.7 69 *** Got signal Terminated *** 2024-05-03T09:00:05.700 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:05 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:05.697+0000 7fc7e2417700 -1 osd.7 69 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-03T09:00:05.977 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:05 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:05.861+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T09:00:05.977 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:05 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:05.862+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T09:00:05.977 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:05 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:05.862+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:26.437726+0000 front 2024-05-03T08:59:26.438420+0000 (oldest deadline 2024-05-03T08:59:52.337769+0000) 2024-05-03T09:00:05.977 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:05 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:05.862+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6830 osd.3 since back 2024-05-03T08:59:38.739456+0000 front 2024-05-03T08:59:38.739500+0000 (oldest deadline 2024-05-03T09:00:03.439552+0000) 2024-05-03T09:00:06.348 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.61 MB/sec execute 78 sec latency 57321.029 ms 2024-05-03T09:00:07.227 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:06 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:06.823+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T09:00:07.227 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:06 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:06.823+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T09:00:07.227 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:06 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:06.823+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:26.437726+0000 front 2024-05-03T08:59:26.438420+0000 (oldest deadline 2024-05-03T08:59:52.337769+0000) 2024-05-03T09:00:07.227 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:06 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:06.823+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6830 osd.3 since back 2024-05-03T08:59:38.739456+0000 front 2024-05-03T08:59:38.739500+0000 (oldest deadline 2024-05-03T09:00:03.439552+0000) 2024-05-03T09:00:07.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.56 MB/sec execute 79 sec latency 58321.078 ms 2024-05-03T09:00:08.128 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:07 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:07.871+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T09:00:08.128 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:07 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:07.871+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T09:00:08.129 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:07 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:07.871+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:26.437726+0000 front 2024-05-03T08:59:26.438420+0000 (oldest deadline 2024-05-03T08:59:52.337769+0000) 2024-05-03T09:00:08.129 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:07 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:07.871+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6830 osd.3 since back 2024-05-03T08:59:38.739456+0000 front 2024-05-03T08:59:38.739500+0000 (oldest deadline 2024-05-03T09:00:03.439552+0000) 2024-05-03T09:00:08.347 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.52 MB/sec execute 80 sec latency 59321.120 ms 2024-05-03T09:00:09.227 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:08 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:08.864+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T09:00:09.227 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:08 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:08.864+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T09:00:09.227 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:08 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:08.864+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:26.437726+0000 front 2024-05-03T08:59:26.438420+0000 (oldest deadline 2024-05-03T08:59:52.337769+0000) 2024-05-03T09:00:09.227 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:08 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:08.864+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6830 osd.3 since back 2024-05-03T08:59:38.739456+0000 front 2024-05-03T08:59:38.739500+0000 (oldest deadline 2024-05-03T09:00:03.439552+0000) 2024-05-03T09:00:09.228 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:08 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:08.864+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.123:6804 osd.4 since back 2024-05-03T08:59:43.439956+0000 front 2024-05-03T08:59:43.440000+0000 (oldest deadline 2024-05-03T09:00:08.140035+0000) 2024-05-03T09:00:09.348 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.48 MB/sec execute 81 sec latency 60321.178 ms 2024-05-03T09:00:10.227 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:09 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:09.845+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6806 osd.0 since back 2024-05-03T08:59:14.735936+0000 front 2024-05-03T08:59:14.735921+0000 (oldest deadline 2024-05-03T08:59:40.636181+0000) 2024-05-03T09:00:10.227 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:09 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:09.845+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6814 osd.1 since back 2024-05-03T08:59:21.137072+0000 front 2024-05-03T08:59:21.137123+0000 (oldest deadline 2024-05-03T08:59:46.437223+0000) 2024-05-03T09:00:10.227 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:09 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:09.845+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6822 osd.2 since back 2024-05-03T08:59:26.437726+0000 front 2024-05-03T08:59:26.438420+0000 (oldest deadline 2024-05-03T08:59:52.337769+0000) 2024-05-03T09:00:10.227 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:09 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:09.845+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.32:6830 osd.3 since back 2024-05-03T08:59:38.739456+0000 front 2024-05-03T08:59:38.739500+0000 (oldest deadline 2024-05-03T09:00:03.439552+0000) 2024-05-03T09:00:10.228 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:09 smithi123 ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7[37761]: 2024-05-03T09:00:09.845+0000 7fc7dedb2700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.123:6804 osd.4 since back 2024-05-03T08:59:43.439956+0000 front 2024-05-03T08:59:43.440000+0000 (oldest deadline 2024-05-03T09:00:08.140035+0000) 2024-05-03T09:00:10.348 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.43 MB/sec execute 82 sec latency 61321.218 ms 2024-05-03T09:00:11.348 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.39 MB/sec execute 83 sec latency 62321.258 ms 2024-05-03T09:00:11.477 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:11 smithi123 podman[41285]: 2024-05-03 09:00:11.222590489 +0000 UTC m=+5.682511053 container died 40d54616bfbe57d7319f122b7036ac55c52fb9c5b5885fe8bd16582f84f999c9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vcs-type=git, architecture=x86_64, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, ceph=True, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.openshift.expose-services=) 2024-05-03T09:00:11.832 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:11 smithi123 podman[41285]: 2024-05-03 09:00:11.572095875 +0000 UTC m=+6.032016431 container cleanup 40d54616bfbe57d7319f122b7036ac55c52fb9c5b5885fe8bd16582f84f999c9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, version=8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8) 2024-05-03T09:00:11.832 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:11 smithi123 bash[41285]: ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7 2024-05-03T09:00:11.832 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:11 smithi123 bash[41325]: Error: no container with name or ID "ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd.7" found: no such container 2024-05-03T09:00:12.137 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:11 smithi123 podman[41314]: 2024-05-03 09:00:11.831395446 +0000 UTC m=+0.611074453 container remove 40d54616bfbe57d7319f122b7036ac55c52fb9c5b5885fe8bd16582f84f999c9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, ceph=True, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container) 2024-05-03T09:00:12.137 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:12 smithi123 podman[41389]: 2024-05-03 09:00:11.914145001 +0000 UTC m=+0.019160891 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-03T09:00:12.137 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:12 smithi123 podman[41389]: 2024-05-03 09:00:12.015455432 +0000 UTC m=+0.120471308 container create 756c281942cf5bdbc071928634cbd79aa89e6e5f3c7060418e949815ca967f86 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7-deactivate, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-05-03T09:00:12.348 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.35 MB/sec execute 84 sec latency 63321.302 ms 2024-05-03T09:00:12.477 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:12 smithi123 podman[41389]: 2024-05-03 09:00:12.260458309 +0000 UTC m=+0.365474186 container init 756c281942cf5bdbc071928634cbd79aa89e6e5f3c7060418e949815ca967f86 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, release=754, distribution-scope=public, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8) 2024-05-03T09:00:12.477 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:12 smithi123 podman[41389]: 2024-05-03 09:00:12.263704181 +0000 UTC m=+0.368720066 container start 756c281942cf5bdbc071928634cbd79aa89e6e5f3c7060418e949815ca967f86 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, version=8, io.openshift.expose-services=, release=754, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, maintainer=Guillaume Abrioux , distribution-scope=public, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, vcs-type=git, RELEASE=HEAD, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-05-03T09:00:12.477 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:12 smithi123 podman[41389]: 2024-05-03 09:00:12.342737917 +0000 UTC m=+0.447753798 container attach 756c281942cf5bdbc071928634cbd79aa89e6e5f3c7060418e949815ca967f86 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, vcs-type=git, architecture=x86_64, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., RELEASE=HEAD, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, distribution-scope=public, name=centos-stream) 2024-05-03T09:00:12.821 INFO:journalctl@ceph.osd.7.smithi123.stdout:May 03 09:00:12 smithi123 podman[41389]: 2024-05-03 09:00:12.54347183 +0000 UTC m=+0.648487708 container died 756c281942cf5bdbc071928634cbd79aa89e6e5f3c7060418e949815ca967f86 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-df63fec2-0929-11ef-bc97-c7b262605968-osd-7-deactivate, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., RELEASE=HEAD, distribution-scope=public, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, architecture=x86_64, ceph=True) 2024-05-03T09:00:13.095 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-df63fec2-0929-11ef-bc97-c7b262605968@osd.7.service' 2024-05-03T09:00:13.134 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-03T09:00:13.134 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-05-03T09:00:13.134 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid df63fec2-0929-11ef-bc97-c7b262605968 --force --keep-logs 2024-05-03T09:00:13.348 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.31 MB/sec execute 85 sec latency 64321.342 ms 2024-05-03T09:00:14.348 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.27 MB/sec execute 86 sec latency 65321.374 ms 2024-05-03T09:00:15.348 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.24 MB/sec execute 87 sec latency 66321.830 ms 2024-05-03T09:00:16.348 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.20 MB/sec execute 88 sec latency 67321.860 ms 2024-05-03T09:00:17.348 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.16 MB/sec execute 89 sec latency 68321.904 ms 2024-05-03T09:00:18.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.13 MB/sec execute 90 sec latency 69321.946 ms 2024-05-03T09:00:19.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.09 MB/sec execute 91 sec latency 70322.004 ms 2024-05-03T09:00:20.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.06 MB/sec execute 92 sec latency 71322.053 ms 2024-05-03T09:00:21.348 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.03 MB/sec execute 93 sec latency 72322.098 ms 2024-05-03T09:00:22.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 3.00 MB/sec execute 94 sec latency 73322.139 ms 2024-05-03T09:00:23.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.96 MB/sec execute 95 sec latency 74322.175 ms 2024-05-03T09:00:24.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.93 MB/sec execute 96 sec latency 75322.230 ms 2024-05-03T09:00:25.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.90 MB/sec execute 97 sec latency 76322.284 ms 2024-05-03T09:00:26.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.87 MB/sec execute 98 sec latency 77322.330 ms 2024-05-03T09:00:27.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.84 MB/sec execute 99 sec latency 78322.371 ms 2024-05-03T09:00:28.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.82 MB/sec execute 100 sec latency 79322.426 ms 2024-05-03T09:00:29.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.79 MB/sec execute 101 sec latency 80322.487 ms 2024-05-03T09:00:30.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.76 MB/sec execute 102 sec latency 81322.530 ms 2024-05-03T09:00:31.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.73 MB/sec execute 103 sec latency 82322.581 ms 2024-05-03T09:00:32.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.71 MB/sec execute 104 sec latency 83322.627 ms 2024-05-03T09:00:33.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.68 MB/sec execute 105 sec latency 84322.673 ms 2024-05-03T09:00:34.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.66 MB/sec execute 106 sec latency 85322.690 ms 2024-05-03T09:00:35.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.63 MB/sec execute 107 sec latency 86322.738 ms 2024-05-03T09:00:36.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.61 MB/sec execute 108 sec latency 87322.774 ms 2024-05-03T09:00:37.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.58 MB/sec execute 109 sec latency 88322.800 ms 2024-05-03T09:00:38.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.56 MB/sec execute 110 sec latency 89322.818 ms 2024-05-03T09:00:39.350 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.54 MB/sec execute 111 sec latency 90322.849 ms 2024-05-03T09:00:39.558 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid df63fec2-0929-11ef-bc97-c7b262605968 --force --keep-logs 2024-05-03T09:00:40.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.51 MB/sec execute 112 sec latency 91322.896 ms 2024-05-03T09:00:41.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.49 MB/sec execute 113 sec latency 92322.939 ms 2024-05-03T09:00:42.349 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.47 MB/sec execute 114 sec latency 93322.984 ms 2024-05-03T09:00:43.350 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.45 MB/sec execute 115 sec latency 94323.102 ms 2024-05-03T09:00:44.350 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.43 MB/sec execute 116 sec latency 95323.208 ms 2024-05-03T09:00:45.350 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.41 MB/sec execute 117 sec latency 96323.312 ms 2024-05-03T09:00:46.350 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.39 MB/sec execute 118 sec latency 97323.411 ms 2024-05-03T09:00:47.350 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.37 MB/sec execute 119 sec latency 98323.529 ms 2024-05-03T09:00:48.350 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.35 MB/sec execute 120 sec latency 99323.661 ms 2024-05-03T09:00:49.350 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.33 MB/sec execute 121 sec latency 100323.713 ms 2024-05-03T09:00:50.353 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.31 MB/sec execute 122 sec latency 101323.766 ms 2024-05-03T09:00:51.350 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.29 MB/sec execute 123 sec latency 102323.883 ms 2024-05-03T09:00:52.350 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.27 MB/sec execute 124 sec latency 103323.991 ms 2024-05-03T09:00:53.351 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.25 MB/sec execute 125 sec latency 104324.118 ms 2024-05-03T09:00:54.351 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.23 MB/sec execute 126 sec latency 105324.237 ms 2024-05-03T09:00:55.351 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.22 MB/sec execute 127 sec latency 106324.332 ms 2024-05-03T09:00:56.351 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.20 MB/sec execute 128 sec latency 107324.458 ms 2024-05-03T09:00:57.351 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.18 MB/sec execute 129 sec latency 108324.562 ms 2024-05-03T09:00:58.190 DEBUG:teuthology.orchestra.run.smithi032:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-03T09:00:58.224 DEBUG:teuthology.orchestra.run.smithi123:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-03T09:00:58.249 INFO:tasks.cephadm:Archiving crash dumps... 2024-05-03T09:00:58.251 DEBUG:teuthology.misc:Transferring archived files from smithi032:/var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/crash to /home/teuthworker/archive/gabrioux-2024-05-03_06:31:58-orch:cephadm-wip-guits-testing-2024-05-02-0628-distro-default-smithi/7687215/remote/smithi032/crash 2024-05-03T09:00:58.252 DEBUG:teuthology.orchestra.run.smithi032:> sudo tar c -f - -C /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/crash -- . 2024-05-03T09:00:58.302 INFO:teuthology.orchestra.run.smithi032.stderr:tar: /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/crash: Cannot open: No such file or directory 2024-05-03T09:00:58.302 INFO:teuthology.orchestra.run.smithi032.stderr:tar: Error is not recoverable: exiting now 2024-05-03T09:00:58.304 DEBUG:teuthology.misc:Transferring archived files from smithi123:/var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/crash to /home/teuthworker/archive/gabrioux-2024-05-03_06:31:58-orch:cephadm-wip-guits-testing-2024-05-02-0628-distro-default-smithi/7687215/remote/smithi123/crash 2024-05-03T09:00:58.305 DEBUG:teuthology.orchestra.run.smithi123:> sudo tar c -f - -C /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/crash -- . 2024-05-03T09:00:58.343 INFO:teuthology.orchestra.run.smithi123.stderr:tar: /var/lib/ceph/df63fec2-0929-11ef-bc97-c7b262605968/crash: Cannot open: No such file or directory 2024-05-03T09:00:58.343 INFO:teuthology.orchestra.run.smithi123.stderr:tar: Error is not recoverable: exiting now 2024-05-03T09:00:58.344 INFO:tasks.cephadm:Checking cluster log for badness... 2024-05-03T09:00:58.344 DEBUG:teuthology.orchestra.run.smithi032:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-05-03T09:00:58.351 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.17 MB/sec execute 130 sec latency 109324.609 ms 2024-05-03T09:00:58.373 INFO:tasks.cephadm:Compressing logs... 2024-05-03T09:00:58.373 DEBUG:teuthology.orchestra.run.smithi032:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-05-03T09:00:58.416 DEBUG:teuthology.orchestra.run.smithi123:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-05-03T09:00:58.449 INFO:teuthology.orchestra.run.smithi032.stderr:find: ‘/var/log/rbd-target-api’gzip: No such file or directory -5 2024-05-03T09:00:58.449 INFO:teuthology.orchestra.run.smithi032.stderr: --verbose -- /var/log/ceph/cephadm.log 2024-05-03T09:00:58.449 INFO:teuthology.orchestra.run.smithi032.stderr:gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph.log 2024-05-03T09:00:58.449 INFO:teuthology.orchestra.run.smithi032.stderr:gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.2.log 2024-05-03T09:00:58.449 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/cephadm.log: /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph.audit.log 2024-05-03T09:00:58.450 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.3.log 2024-05-03T09:00:58.450 INFO:teuthology.orchestra.run.smithi032.stderr:gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-mon.smithi032.log 2024-05-03T09:00:58.450 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph.audit.log: /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-mgr.smithi032.umllay.log 2024-05-03T09:00:58.450 INFO:teuthology.orchestra.run.smithi032.stderr: 86.7% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph.log.gz 2024-05-03T09:00:58.450 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-mon.smithi032.log: gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.1.log 2024-05-03T09:00:58.450 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-mgr.smithi032.umllay.log: gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-mds.foofs.smithi032.axmmtz.log 2024-05-03T09:00:58.450 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.0.log 2024-05-03T09:00:58.450 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-mds.foofs.smithi032.axmmtz.log: 80.2% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-mds.foofs.smithi032.axmmtz.log.gz 2024-05-03T09:00:58.451 INFO:teuthology.orchestra.run.smithi032.stderr: 90.8% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph.audit.log.gz 2024-05-03T09:00:58.451 INFO:teuthology.orchestra.run.smithi032.stderr:gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph.cephadm.log 2024-05-03T09:00:58.451 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-volume.log 2024-05-03T09:00:58.451 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph.cephadm.log: 83.1% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph.cephadm.log.gz 2024-05-03T09:00:58.452 INFO:teuthology.orchestra.run.smithi123.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose --: No such file or directory /var/log/ceph/cephadm.log 2024-05-03T09:00:58.452 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-05-03T09:00:58.453 INFO:teuthology.orchestra.run.smithi123.stderr:gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph.log 2024-05-03T09:00:58.453 INFO:teuthology.orchestra.run.smithi123.stderr:gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph.audit.log 2024-05-03T09:00:58.453 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/cephadm.log: /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.7.log 2024-05-03T09:00:58.453 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.5.log 2024-05-03T09:00:58.453 INFO:teuthology.orchestra.run.smithi123.stderr:gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-mgr.smithi123.xrxinh.log 2024-05-03T09:00:58.453 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.7.log: /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.4.log 2024-05-03T09:00:58.453 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-mgr.smithi123.xrxinh.log: 86.7%gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-mon.smithi123.log 2024-05-03T09:00:58.453 INFO:teuthology.orchestra.run.smithi123.stderr: -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph.log.gz 2024-05-03T09:00:58.454 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.6.log 2024-05-03T09:00:58.454 INFO:teuthology.orchestra.run.smithi123.stderr: 91.0% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph.audit.log.gz 2024-05-03T09:00:58.454 INFO:teuthology.orchestra.run.smithi123.stderr:gzip -5 --verbose/var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-mon.smithi123.log: -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-mds.foofs.smithi123.gtrlts.log 2024-05-03T09:00:58.454 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph.cephadm.log 2024-05-03T09:00:58.454 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-mds.foofs.smithi123.gtrlts.log: 60.3% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-mds.foofs.smithi123.gtrlts.log.gz 2024-05-03T09:00:58.454 INFO:teuthology.orchestra.run.smithi123.stderr:gzip -5 --verbose -- /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-volume.log 2024-05-03T09:00:58.454 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph.cephadm.log: 82.3% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph.cephadm.log.gz 2024-05-03T09:00:58.454 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-volume.log: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-05-03T09:00:58.454 INFO:teuthology.orchestra.run.smithi123.stderr: 92.3% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-mgr.smithi123.xrxinh.log.gz 2024-05-03T09:00:58.455 INFO:teuthology.orchestra.run.smithi123.stderr: 90.7% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-volume.log.gz 2024-05-03T09:00:58.455 INFO:teuthology.orchestra.run.smithi032.stderr:/var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-volume.log: 90.8% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-volume.log.gz 2024-05-03T09:00:58.455 INFO:teuthology.orchestra.run.smithi032.stderr: 91.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-05-03T09:00:58.539 INFO:teuthology.orchestra.run.smithi123.stderr: 92.6% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-mon.smithi123.log.gz 2024-05-03T09:00:58.577 INFO:teuthology.orchestra.run.smithi032.stderr: 88.7% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-mgr.smithi032.umllay.log.gz 2024-05-03T09:00:58.941 INFO:teuthology.orchestra.run.smithi032.stderr: 91.9% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-mon.smithi032.log.gz 2024-05-03T09:00:59.351 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.15 MB/sec execute 131 sec latency 110324.644 ms 2024-05-03T09:01:00.351 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.13 MB/sec execute 132 sec latency 111324.691 ms 2024-05-03T09:01:01.098 INFO:teuthology.orchestra.run.smithi032.stderr: 94.6% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.2.log.gz 2024-05-03T09:01:01.351 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.12 MB/sec execute 133 sec latency 112324.733 ms 2024-05-03T09:01:02.351 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.10 MB/sec execute 134 sec latency 113324.783 ms 2024-05-03T09:01:02.383 INFO:teuthology.orchestra.run.smithi123.stderr: 94.3% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.6.log.gz 2024-05-03T09:01:02.524 INFO:teuthology.orchestra.run.smithi123.stderr: 94.1% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.4.log.gz 2024-05-03T09:01:03.208 INFO:teuthology.orchestra.run.smithi032.stderr: 94.3% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.1.log.gz 2024-05-03T09:01:03.351 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.09 MB/sec execute 135 sec latency 114324.836 ms 2024-05-03T09:01:03.444 INFO:teuthology.orchestra.run.smithi032.stderr: 94.2% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.0.log.gz 2024-05-03T09:01:03.545 INFO:teuthology.orchestra.run.smithi032.stderr: 94.1% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.3.log.gz 2024-05-03T09:01:03.548 INFO:teuthology.orchestra.run.smithi032.stderr: 2024-05-03T09:01:03.548 INFO:teuthology.orchestra.run.smithi032.stderr:real 0m5.119s 2024-05-03T09:01:03.548 INFO:teuthology.orchestra.run.smithi032.stderr:user 0m17.565s 2024-05-03T09:01:03.548 INFO:teuthology.orchestra.run.smithi032.stderr:sys 0m0.619s 2024-05-03T09:01:03.686 INFO:teuthology.orchestra.run.smithi123.stderr: 94.3% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.5.log.gz 2024-05-03T09:01:04.351 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.07 MB/sec execute 136 sec latency 115324.973 ms 2024-05-03T09:01:05.352 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.06 MB/sec execute 137 sec latency 116325.113 ms 2024-05-03T09:01:05.626 INFO:teuthology.orchestra.run.smithi123.stderr: 94.2% -- replaced with /var/log/ceph/df63fec2-0929-11ef-bc97-c7b262605968/ceph-osd.7.log.gz 2024-05-03T09:01:05.628 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-05-03T09:01:05.628 INFO:teuthology.orchestra.run.smithi123.stderr:real 0m7.198s 2024-05-03T09:01:05.628 INFO:teuthology.orchestra.run.smithi123.stderr:user 0m19.780s 2024-05-03T09:01:05.628 INFO:teuthology.orchestra.run.smithi123.stderr:sys 0m0.783s 2024-05-03T09:01:05.628 INFO:tasks.cephadm:Archiving logs... 2024-05-03T09:01:05.629 DEBUG:teuthology.misc:Transferring archived files from smithi032:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-05-03_06:31:58-orch:cephadm-wip-guits-testing-2024-05-02-0628-distro-default-smithi/7687215/remote/smithi032/log 2024-05-03T09:01:05.630 DEBUG:teuthology.orchestra.run.smithi032:> sudo tar c -f - -C /var/log/ceph -- . 2024-05-03T09:01:06.367 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.04 MB/sec execute 138 sec latency 117325.180 ms 2024-05-03T09:01:07.382 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.03 MB/sec execute 139 sec latency 118325.267 ms 2024-05-03T09:01:07.978 DEBUG:teuthology.misc:Transferring archived files from smithi123:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-05-03_06:31:58-orch:cephadm-wip-guits-testing-2024-05-02-0628-distro-default-smithi/7687215/remote/smithi123/log 2024-05-03T09:01:07.979 DEBUG:teuthology.orchestra.run.smithi123:> sudo tar c -f - -C /var/log/ceph -- . 2024-05-03T09:01:08.355 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.01 MB/sec execute 140 sec latency 119325.394 ms 2024-05-03T09:01:09.357 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 2.00 MB/sec execute 141 sec latency 120325.518 ms 2024-05-03T09:01:10.366 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 1.98 MB/sec execute 142 sec latency 121325.672 ms 2024-05-03T09:01:10.787 INFO:tasks.cephadm:Removing cluster... 2024-05-03T09:01:10.788 DEBUG:teuthology.orchestra.run.smithi032:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid df63fec2-0929-11ef-bc97-c7b262605968 --force 2024-05-03T09:01:11.103 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid df63fec2-0929-11ef-bc97-c7b262605968 --force 2024-05-03T09:01:11.352 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 1.97 MB/sec execute 143 sec latency 122325.710 ms 2024-05-03T09:01:11.413 INFO:tasks.cephadm:Removing cephadm ... 2024-05-03T09:01:11.414 DEBUG:teuthology.orchestra.run.smithi032:> rm -rf /home/ubuntu/cephtest/cephadm 2024-05-03T09:01:11.430 DEBUG:teuthology.orchestra.run.smithi123:> rm -rf /home/ubuntu/cephtest/cephadm 2024-05-03T09:01:11.445 INFO:tasks.cephadm:Teardown complete 2024-05-03T09:01:11.445 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-05-03T09:01:11.456 INFO:teuthology.task.clock:Checking final clock skew... 2024-05-03T09:01:11.456 DEBUG:teuthology.orchestra.run.smithi032:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-03T09:01:11.473 DEBUG:teuthology.orchestra.run.smithi123:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-03T09:01:11.485 INFO:teuthology.orchestra.run.smithi032.stderr:bash: line 1: ntpq: command not found 2024-05-03T09:01:11.490 INFO:teuthology.orchestra.run.smithi032.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-03T09:01:11.490 INFO:teuthology.orchestra.run.smithi032.stdout:=============================================================================== 2024-05-03T09:01:11.490 INFO:teuthology.orchestra.run.smithi032.stdout:^* hv01.front.sepia.ceph.com 3 6 377 190 +493us[ +496us] +/- 44ms 2024-05-03T09:01:11.491 INFO:teuthology.orchestra.run.smithi032.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 60 +306us[ +306us] +/- 70ms 2024-05-03T09:01:11.491 INFO:teuthology.orchestra.run.smithi032.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 62 -1352us[-1352us] +/- 90ms 2024-05-03T09:01:11.491 INFO:teuthology.orchestra.run.smithi032.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-05-03T09:01:11.499 INFO:teuthology.orchestra.run.smithi123.stderr:bash: line 1: ntpq: command not found 2024-05-03T09:01:11.503 INFO:teuthology.orchestra.run.smithi123.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-03T09:01:11.503 INFO:teuthology.orchestra.run.smithi123.stdout:=============================================================================== 2024-05-03T09:01:11.504 INFO:teuthology.orchestra.run.smithi123.stdout:^* hv01.front.sepia.ceph.com 3 6 377 62 +467us[ +463us] +/- 46ms 2024-05-03T09:01:11.504 INFO:teuthology.orchestra.run.smithi123.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 62 +309us[ +309us] +/- 69ms 2024-05-03T09:01:11.504 INFO:teuthology.orchestra.run.smithi123.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 64 -1342us[-1346us] +/- 90ms 2024-05-03T09:01:11.504 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-05-03T09:01:11.505 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-05-03T09:01:11.515 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-05-03T09:01:11.516 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-05-03T09:01:11.526 DEBUG:teuthology.orchestra.run.smithi032:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-05-03T09:01:11.604 DEBUG:teuthology.orchestra.run.smithi123:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-05-03T09:01:11.666 DEBUG:teuthology.orchestra.run.smithi032:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-05-03T09:01:11.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-03T09:01:11.695 DEBUG:teuthology.orchestra.run.smithi123:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-05-03T09:01:11.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-03T09:01:11.722 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-05-03T09:01:11.731 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-05-03T09:01:11.741 INFO:teuthology.task.internal:Duration was 1016.465377 seconds 2024-05-03T09:01:11.742 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-05-03T09:01:11.753 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-05-03T09:01:11.753 DEBUG:teuthology.orchestra.run.smithi032:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-05-03T09:01:11.755 DEBUG:teuthology.orchestra.run.smithi123:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-05-03T09:01:11.791 INFO:teuthology.orchestra.run.smithi032.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-03T09:01:11.799 INFO:teuthology.orchestra.run.smithi123.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-03T09:01:12.015 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-05-03T09:01:12.015 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi032.front.sepia.ceph.com 2024-05-03T09:01:12.016 DEBUG:teuthology.orchestra.run.smithi032:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-05-03T09:01:12.038 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi123.front.sepia.ceph.com 2024-05-03T09:01:12.038 DEBUG:teuthology.orchestra.run.smithi123:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-05-03T09:01:12.080 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-05-03T09:01:12.081 DEBUG:teuthology.orchestra.run.smithi032:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-05-03T09:01:12.083 DEBUG:teuthology.orchestra.run.smithi123:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-05-03T09:01:12.145 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-05-03T09:01:12.145 DEBUG:teuthology.orchestra.run.smithi032:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-05-03T09:01:12.151 DEBUG:teuthology.orchestra.run.smithi123:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-05-03T09:01:12.352 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 1.96 MB/sec execute 144 sec latency 123325.758 ms 2024-05-03T09:01:13.352 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 1.94 MB/sec execute 145 sec latency 124325.814 ms 2024-05-03T09:01:14.352 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 1.93 MB/sec execute 146 sec latency 125325.870 ms 2024-05-03T09:01:15.352 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 1.92 MB/sec execute 147 sec latency 126325.911 ms 2024-05-03T09:01:16.352 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 1.90 MB/sec execute 148 sec latency 127325.950 ms 2024-05-03T09:01:17.281 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-05-03T09:01:17.292 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-05-03T09:01:17.292 DEBUG:teuthology.orchestra.run.smithi032:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-05-03T09:01:17.316 DEBUG:teuthology.orchestra.run.smithi123:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-05-03T09:01:17.345 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-05-03T09:01:17.361 DEBUG:teuthology.orchestra.run.smithi032:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-05-03T09:01:17.362 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 1.89 MB/sec execute 149 sec latency 128326.020 ms 2024-05-03T09:01:17.364 DEBUG:teuthology.orchestra.run.smithi123:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-05-03T09:01:17.384 INFO:teuthology.orchestra.run.smithi032.stdout:kernel.core_pattern = core 2024-05-03T09:01:17.411 INFO:teuthology.orchestra.run.smithi123.stdout:kernel.core_pattern = core 2024-05-03T09:01:17.470 DEBUG:teuthology.orchestra.run.smithi032:> test -e /home/ubuntu/cephtest/archive/coredump 2024-05-03T09:01:17.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-03T09:01:17.525 DEBUG:teuthology.orchestra.run.smithi123:> test -e /home/ubuntu/cephtest/archive/coredump 2024-05-03T09:01:17.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-03T09:01:17.541 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-05-03T09:01:17.555 INFO:teuthology.task.internal:Transferring archived files... 2024-05-03T09:01:17.556 DEBUG:teuthology.misc:Transferring archived files from smithi032:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-05-03_06:31:58-orch:cephadm-wip-guits-testing-2024-05-02-0628-distro-default-smithi/7687215/remote/smithi032 2024-05-03T09:01:17.556 DEBUG:teuthology.orchestra.run.smithi032:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-05-03T09:01:17.614 DEBUG:teuthology.misc:Transferring archived files from smithi123:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-05-03_06:31:58-orch:cephadm-wip-guits-testing-2024-05-02-0628-distro-default-smithi/7687215/remote/smithi123 2024-05-03T09:01:17.615 DEBUG:teuthology.orchestra.run.smithi123:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-05-03T09:01:17.651 INFO:teuthology.task.internal:Removing archive directory... 2024-05-03T09:01:17.651 DEBUG:teuthology.orchestra.run.smithi032:> rm -rf -- /home/ubuntu/cephtest/archive 2024-05-03T09:01:17.654 DEBUG:teuthology.orchestra.run.smithi123:> rm -rf -- /home/ubuntu/cephtest/archive 2024-05-03T09:01:17.701 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-05-03T09:01:17.714 INFO:teuthology.task.internal:Not uploading archives. 2024-05-03T09:01:17.715 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-05-03T09:01:17.732 INFO:teuthology.task.internal:Tidying up after the test... 2024-05-03T09:01:17.732 DEBUG:teuthology.orchestra.run.smithi032:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-05-03T09:01:17.734 DEBUG:teuthology.orchestra.run.smithi123:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-05-03T09:01:17.747 INFO:teuthology.orchestra.run.smithi032.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 May 3 09:01 /home/ubuntu/cephtest 2024-05-03T09:01:17.756 INFO:teuthology.orchestra.run.smithi123.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 May 3 09:01 /home/ubuntu/cephtest 2024-05-03T09:01:17.757 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-05-03T09:01:17.771 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-05-03T09:01:17.806 INFO:teuthology.nuke:Checking targets against current locks 2024-05-03T09:01:17.838 DEBUG:teuthology.nuke:shortname: smithi032 2024-05-03T09:01:17.838 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-05-03T09:01:17.860 DEBUG:teuthology.nuke:shortname: smithi123 2024-05-03T09:01:17.861 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-05-03T09:01:17.864 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi032.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-05-03_06:31:58-orch:cephadm-wip-guits-testing-2024-05-02-0628-distro-default-smithi/7687215', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-03 08:38:57.932821', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEYrsD74lBa50O7lCv8p0GIMhm9Xi+BbpVGMVX9pOlxGLVWXvq1FiPFSf/r9uExnjfKzFofHp+R3y0s3PiB/8H0='} 2024-05-03T09:01:17.885 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi123.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-05-03_06:31:58-orch:cephadm-wip-guits-testing-2024-05-02-0628-distro-default-smithi/7687215', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-03 08:38:57.931156', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIZAxocJHzqm+vJfluTD1dI9ijqhNKBoFjLanVcpN0pIto+9rKftW5yCQJ13k8hZ/j02C2n+cH+NEG0lbFvvfNE='} 2024-05-03T09:01:17.913 INFO:teuthology.orchestra.console.smithi032:Power off 2024-05-03T09:01:17.913 DEBUG:teuthology.orchestra.console.smithi032:pexpect command: ipmitool -H smithi032.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-05-03T09:01:17.944 INFO:teuthology.orchestra.console.smithi123:Power off 2024-05-03T09:01:17.944 DEBUG:teuthology.orchestra.console.smithi123:pexpect command: ipmitool -H smithi123.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-05-03T09:01:17.954 DEBUG:teuthology.orchestra.console.smithi032:power off output: Chassis Power Control: Down/Off 2024-05-03T09:01:17.955 DEBUG:teuthology.orchestra.console.smithi032:pexpect command: ipmitool -H smithi032.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-03T09:01:17.968 DEBUG:teuthology.orchestra.console.smithi123:power off output: Chassis Power Control: Down/Off 2024-05-03T09:01:17.969 DEBUG:teuthology.orchestra.console.smithi123:pexpect command: ipmitool -H smithi123.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-03T09:01:17.978 DEBUG:teuthology.orchestra.console.smithi032:check power output: Chassis Power is on 2024-05-03T09:01:17.994 DEBUG:teuthology.orchestra.console.smithi123:check power output: Chassis Power is on 2024-05-03T09:01:18.353 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 1.88 MB/sec execute 150 sec latency 129326.137 ms 2024-05-03T09:01:19.353 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 1.86 MB/sec execute 151 sec latency 130326.254 ms 2024-05-03T09:01:20.353 INFO:teuthology.orchestra.run.smithi032.stdout: 5 84792 1.85 MB/sec execute 152 sec latency 131326.387 ms 2024-05-03T09:01:21.093 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-03T09:01:21.980 DEBUG:teuthology.orchestra.console.smithi032:pexpect command: ipmitool -H smithi032.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-03T09:01:21.995 DEBUG:teuthology.orchestra.console.smithi123:pexpect command: ipmitool -H smithi123.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-03T09:01:22.092 DEBUG:teuthology.orchestra.console.smithi032:check power output: Chassis Power is on 2024-05-03T09:01:22.105 DEBUG:teuthology.orchestra.console.smithi123:check power output: Chassis Power is on 2024-05-03T09:01:26.094 DEBUG:teuthology.orchestra.console.smithi032:pexpect command: ipmitool -H smithi032.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-03T09:01:26.106 DEBUG:teuthology.orchestra.console.smithi123:pexpect command: ipmitool -H smithi123.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-03T09:01:26.204 DEBUG:teuthology.orchestra.console.smithi032:check power output: Chassis Power is off 2024-05-03T09:01:26.216 DEBUG:teuthology.orchestra.console.smithi123:check power output: Chassis Power is off 2024-05-03T09:01:26.305 INFO:teuthology.orchestra.console.smithi032:Power off completed 2024-05-03T09:01:26.317 INFO:teuthology.orchestra.console.smithi123:Power off completed 2024-05-03T09:01:26.422 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1016.4653766155243 failure_reason: 'Command failed on smithi032 with status 1: ''sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid df63fec2-0929-11ef-bc97-c7b262605968 -e sha1=f8e6d2d721bc96a93915917c59bc3d287d57263d -- bash -c \''ceph versions | jq -e \''"\''"\''.overall | keys\''"\''"\'' | grep $sha1\''''' owner: scheduled_gabrioux@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=b75a8212441840eea39e7eba82a24b72 status: fail success: false 2024-05-03T09:01:26.422 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-05-03T09:01:26.507 INFO:teuthology.run:FAIL