2024-04-02T14:38:40.337 INFO:root:teuthology version: 0.0.1.dev262+ge691533 2024-04-02T14:38:40.337 DEBUG:teuthology.run:Teuthology command: teuthology --description orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} --verbose --archive /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636013 --name gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi --owner scheduled_gabrioux@teuthology -- /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636013/orig.config.yaml 2024-04-02T14:38:40.374 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-02T14:38:40.459 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636013 branch: wip-guits-testing-2024-04-02-0703 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: gabrioux@ibm.com first_in_suite: false job_id: '7636013' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-guits-testing-2024-04-02-0703 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 3f3f3ed992429b85be729743ad3b39d52dd90744 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 3f3f3ed992429b85be729743ad3b39d52dd90744 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: wip-guits-testing-2024-04-02-0703 sha1: 3f3f3ed992429b85be729743ad3b39d52dd90744 owner: scheduled_gabrioux@teuthology priority: 70 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 5637 sha1: 3f3f3ed992429b85be729743ad3b39d52dd90744 sleep_before_teardown: 0 subset: 1/8 suite: orch:cephadm suite_branch: wip-guits-testing-2024-04-02-0703 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 3f3f3ed992429b85be729743ad3b39d52dd90744 targets: smithi122.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGn0D8IanrmG7WQtzHNx+XSHVkFRT/s6VVCHg/pgMOUE28lRsj0l+gwINiAGP0/yZFazAstH4mlJUkLXBhLqPcI= smithi183.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK2vGxYt4xuDcnKqESTnYfqv10YcFkW7XasL+xY1XG+Aa36VQ2XK8sNyNWA7mTDzo6LScCKAv57qmOCoJJ1He0o= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: e691533f9cbb33d85b2187bba20d7102f098636d timestamp: 2024-04-02_14:30:22 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: gabrioux verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-04-02T14:38:40.459 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa; will attempt to use it 2024-04-02T14:38:40.460 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa/tasks 2024-04-02T14:38:40.460 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-02T14:38:40.462 INFO:teuthology.task.internal:Checking packages... 2024-04-02T14:38:40.484 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '3f3f3ed992429b85be729743ad3b39d52dd90744' 2024-04-02T14:38:40.485 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-02T14:38:40.485 INFO:teuthology.packaging:ref: None 2024-04-02T14:38:40.485 INFO:teuthology.packaging:tag: None 2024-04-02T14:38:40.485 INFO:teuthology.packaging:branch: wip-guits-testing-2024-04-02-0703 2024-04-02T14:38:40.485 INFO:teuthology.packaging:sha1: 3f3f3ed992429b85be729743ad3b39d52dd90744 2024-04-02T14:38:40.485 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-guits-testing-2024-04-02-0703 2024-04-02T14:38:40.621 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2592.g3f3f3ed9 2024-04-02T14:38:40.622 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-02T14:38:40.629 INFO:teuthology.task.internal:no buildpackages task found 2024-04-02T14:38:40.629 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-02T14:38:40.670 INFO:teuthology.task.internal:Saving configuration 2024-04-02T14:38:40.684 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-02T14:38:40.715 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-02T14:38:40.741 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi122.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636013', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-02 14:33:40.640965', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGn0D8IanrmG7WQtzHNx+XSHVkFRT/s6VVCHg/pgMOUE28lRsj0l+gwINiAGP0/yZFazAstH4mlJUkLXBhLqPcI='} 2024-04-02T14:38:40.763 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi183.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636013', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-02 14:33:40.639385', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK2vGxYt4xuDcnKqESTnYfqv10YcFkW7XasL+xY1XG+Aa36VQ2XK8sNyNWA7mTDzo6LScCKAv57qmOCoJJ1He0o='} 2024-04-02T14:38:40.763 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-02T14:38:40.770 INFO:teuthology.task.internal:roles: ubuntu@smithi122.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-04-02T14:38:40.770 INFO:teuthology.task.internal:roles: ubuntu@smithi183.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-04-02T14:38:40.770 INFO:teuthology.run_tasks:Running task console_log... 2024-04-02T14:38:40.872 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fe4423a2f70>, signals=[15]) 2024-04-02T14:38:40.872 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-02T14:38:40.878 INFO:teuthology.task.internal:Opening connections... 2024-04-02T14:38:40.879 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi122.front.sepia.ceph.com 2024-04-02T14:38:40.880 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi122.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T14:38:40.994 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi183.front.sepia.ceph.com 2024-04-02T14:38:40.995 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi183.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T14:38:41.081 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-02T14:38:41.087 DEBUG:teuthology.orchestra.run.smithi122:> uname -m 2024-04-02T14:38:41.104 INFO:teuthology.orchestra.run.smithi122.stdout:x86_64 2024-04-02T14:38:41.105 DEBUG:teuthology.orchestra.run.smithi122:> cat /etc/os-release 2024-04-02T14:38:41.162 INFO:teuthology.orchestra.run.smithi122.stdout:NAME="CentOS Stream" 2024-04-02T14:38:41.162 INFO:teuthology.orchestra.run.smithi122.stdout:VERSION="9" 2024-04-02T14:38:41.162 INFO:teuthology.orchestra.run.smithi122.stdout:ID="centos" 2024-04-02T14:38:41.162 INFO:teuthology.orchestra.run.smithi122.stdout:ID_LIKE="rhel fedora" 2024-04-02T14:38:41.162 INFO:teuthology.orchestra.run.smithi122.stdout:VERSION_ID="9" 2024-04-02T14:38:41.162 INFO:teuthology.orchestra.run.smithi122.stdout:PLATFORM_ID="platform:el9" 2024-04-02T14:38:41.162 INFO:teuthology.orchestra.run.smithi122.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-02T14:38:41.162 INFO:teuthology.orchestra.run.smithi122.stdout:ANSI_COLOR="0;31" 2024-04-02T14:38:41.162 INFO:teuthology.orchestra.run.smithi122.stdout:LOGO="fedora-logo-icon" 2024-04-02T14:38:41.162 INFO:teuthology.orchestra.run.smithi122.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-02T14:38:41.163 INFO:teuthology.orchestra.run.smithi122.stdout:HOME_URL="https://centos.org/" 2024-04-02T14:38:41.163 INFO:teuthology.orchestra.run.smithi122.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-02T14:38:41.163 INFO:teuthology.orchestra.run.smithi122.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-02T14:38:41.163 INFO:teuthology.orchestra.run.smithi122.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-02T14:38:41.163 INFO:teuthology.lock.ops:Updating smithi122.front.sepia.ceph.com on lock server 2024-04-02T14:38:41.184 DEBUG:teuthology.orchestra.run.smithi183:> uname -m 2024-04-02T14:38:41.201 INFO:teuthology.orchestra.run.smithi183.stdout:x86_64 2024-04-02T14:38:41.201 DEBUG:teuthology.orchestra.run.smithi183:> cat /etc/os-release 2024-04-02T14:38:41.258 INFO:teuthology.orchestra.run.smithi183.stdout:NAME="CentOS Stream" 2024-04-02T14:38:41.258 INFO:teuthology.orchestra.run.smithi183.stdout:VERSION="9" 2024-04-02T14:38:41.258 INFO:teuthology.orchestra.run.smithi183.stdout:ID="centos" 2024-04-02T14:38:41.258 INFO:teuthology.orchestra.run.smithi183.stdout:ID_LIKE="rhel fedora" 2024-04-02T14:38:41.258 INFO:teuthology.orchestra.run.smithi183.stdout:VERSION_ID="9" 2024-04-02T14:38:41.258 INFO:teuthology.orchestra.run.smithi183.stdout:PLATFORM_ID="platform:el9" 2024-04-02T14:38:41.258 INFO:teuthology.orchestra.run.smithi183.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-02T14:38:41.258 INFO:teuthology.orchestra.run.smithi183.stdout:ANSI_COLOR="0;31" 2024-04-02T14:38:41.258 INFO:teuthology.orchestra.run.smithi183.stdout:LOGO="fedora-logo-icon" 2024-04-02T14:38:41.259 INFO:teuthology.orchestra.run.smithi183.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-02T14:38:41.259 INFO:teuthology.orchestra.run.smithi183.stdout:HOME_URL="https://centos.org/" 2024-04-02T14:38:41.259 INFO:teuthology.orchestra.run.smithi183.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-02T14:38:41.259 INFO:teuthology.orchestra.run.smithi183.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-02T14:38:41.259 INFO:teuthology.orchestra.run.smithi183.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-02T14:38:41.259 INFO:teuthology.lock.ops:Updating smithi183.front.sepia.ceph.com on lock server 2024-04-02T14:38:41.450 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-02T14:38:41.464 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-02T14:38:41.504 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-02T14:38:41.504 DEBUG:teuthology.orchestra.run.smithi122:> test '!' -e /home/ubuntu/cephtest 2024-04-02T14:38:41.506 DEBUG:teuthology.orchestra.run.smithi183:> test '!' -e /home/ubuntu/cephtest 2024-04-02T14:38:41.523 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-02T14:38:41.545 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-02T14:38:41.545 DEBUG:teuthology.orchestra.run.smithi122:> test -z $(ls -A /var/lib/ceph) 2024-04-02T14:38:41.563 DEBUG:teuthology.orchestra.run.smithi183:> test -z $(ls -A /var/lib/ceph) 2024-04-02T14:38:41.672 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-02T14:38:41.731 INFO:teuthology.run_tasks:Running task kernel... 2024-04-02T14:38:41.750 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-04-02T14:38:41.751 INFO:teuthology.task.kernel:config {'host.a': {'kdb': True, 'sha1': 'distro'}, 'host.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-04-02T14:38:41.751 DEBUG:teuthology.orchestra.run.smithi122:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-02T14:38:41.752 DEBUG:teuthology.orchestra.run.smithi183:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-02T14:38:41.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T14:38:41.769 DEBUG:teuthology.orchestra.run.smithi122:> uname -r 2024-04-02T14:38:41.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T14:38:41.771 DEBUG:teuthology.orchestra.run.smithi183:> uname -r 2024-04-02T14:38:41.826 INFO:teuthology.orchestra.run.smithi122.stdout:5.14.0-361.el9.x86_64 2024-04-02T14:38:41.826 INFO:teuthology.task.kernel:Running kernel on smithi122: 5.14.0-361.el9.x86_64 2024-04-02T14:38:41.826 DEBUG:teuthology.orchestra.run.smithi122:> sudo yum install -y kernel 2024-04-02T14:38:41.828 INFO:teuthology.orchestra.run.smithi183.stdout:5.14.0-361.el9.x86_64 2024-04-02T14:38:41.828 INFO:teuthology.task.kernel:Running kernel on smithi183: 5.14.0-361.el9.x86_64 2024-04-02T14:38:41.828 DEBUG:teuthology.orchestra.run.smithi183:> sudo yum install -y kernel 2024-04-02T14:38:43.924 INFO:teuthology.orchestra.run.smithi183.stdout:CentOS-9 - AppStream 12 MB/s | 6.7 MB 00:00 2024-04-02T14:38:44.040 INFO:teuthology.orchestra.run.smithi122.stdout:CentOS-9 - AppStream 12 MB/s | 6.7 MB 00:00 2024-04-02T14:38:45.308 INFO:teuthology.orchestra.run.smithi183.stdout:CentOS-9 - BaseOS 7.1 MB/s | 2.0 MB 00:00 2024-04-02T14:38:45.408 INFO:teuthology.orchestra.run.smithi122.stdout:CentOS-9 - BaseOS 7.4 MB/s | 2.0 MB 00:00 2024-04-02T14:38:46.165 INFO:teuthology.orchestra.run.smithi122.stdout:CentOS Stream 9 - BaseOS 19 MB/s | 8.0 MB 00:00 2024-04-02T14:38:46.565 INFO:teuthology.orchestra.run.smithi183.stdout:CentOS Stream 9 - BaseOS 8.7 MB/s | 8.0 MB 00:00 2024-04-02T14:38:48.133 INFO:teuthology.orchestra.run.smithi122.stdout:CentOS Stream 9 - AppStream 21 MB/s | 19 MB 00:00 2024-04-02T14:38:48.917 INFO:teuthology.orchestra.run.smithi183.stdout:CentOS Stream 9 - AppStream 15 MB/s | 19 MB 00:01 2024-04-02T14:38:52.851 INFO:teuthology.orchestra.run.smithi183.stdout:CentOS Stream 9 - CRB 13 MB/s | 6.1 MB 00:00 2024-04-02T14:38:53.026 INFO:teuthology.orchestra.run.smithi122.stdout:CentOS Stream 9 - CRB 4.6 MB/s | 6.1 MB 00:01 2024-04-02T14:38:54.310 INFO:teuthology.orchestra.run.smithi183.stdout:CentOS Stream 9 - Extras packages 49 kB/s | 16 kB 00:00 2024-04-02T14:38:54.676 INFO:teuthology.orchestra.run.smithi122.stdout:CentOS Stream 9 - Extras packages 32 kB/s | 16 kB 00:00 2024-04-02T14:38:55.961 INFO:teuthology.orchestra.run.smithi183.stdout:Extra Packages for Enterprise Linux 14 MB/s | 21 MB 00:01 2024-04-02T14:39:00.719 INFO:teuthology.orchestra.run.smithi122.stdout:Extra Packages for Enterprise Linux 3.5 MB/s | 21 MB 00:05 2024-04-02T14:39:00.969 INFO:teuthology.orchestra.run.smithi183.stdout:lab-extras 12 kB/s | 1.7 kB 00:00 2024-04-02T14:39:03.209 INFO:teuthology.orchestra.run.smithi183.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-02T14:39:03.209 INFO:teuthology.orchestra.run.smithi183.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-02T14:39:03.302 INFO:teuthology.orchestra.run.smithi183.stdout:Dependencies resolved. 2024-04-02T14:39:03.312 INFO:teuthology.orchestra.run.smithi183.stdout:================================================================================ 2024-04-02T14:39:03.312 INFO:teuthology.orchestra.run.smithi183.stdout: Package Arch Version Repository Size 2024-04-02T14:39:03.312 INFO:teuthology.orchestra.run.smithi183.stdout:================================================================================ 2024-04-02T14:39:03.312 INFO:teuthology.orchestra.run.smithi183.stdout:Installing: 2024-04-02T14:39:03.313 INFO:teuthology.orchestra.run.smithi183.stdout: kernel x86_64 5.14.0-434.el9 CentOS-BaseOS 192 k 2024-04-02T14:39:03.313 INFO:teuthology.orchestra.run.smithi183.stdout:Installing dependencies: 2024-04-02T14:39:03.313 INFO:teuthology.orchestra.run.smithi183.stdout: kernel-core x86_64 5.14.0-434.el9 CentOS-BaseOS 15 M 2024-04-02T14:39:03.313 INFO:teuthology.orchestra.run.smithi183.stdout: kernel-modules x86_64 5.14.0-434.el9 CentOS-BaseOS 34 M 2024-04-02T14:39:03.313 INFO:teuthology.orchestra.run.smithi183.stdout: kernel-modules-core x86_64 5.14.0-434.el9 CentOS-BaseOS 28 M 2024-04-02T14:39:03.313 INFO:teuthology.orchestra.run.smithi183.stdout: 2024-04-02T14:39:03.313 INFO:teuthology.orchestra.run.smithi183.stdout:Transaction Summary 2024-04-02T14:39:03.313 INFO:teuthology.orchestra.run.smithi183.stdout:================================================================================ 2024-04-02T14:39:03.313 INFO:teuthology.orchestra.run.smithi183.stdout:Install 4 Packages 2024-04-02T14:39:03.314 INFO:teuthology.orchestra.run.smithi183.stdout: 2024-04-02T14:39:03.314 INFO:teuthology.orchestra.run.smithi183.stdout:Total download size: 76 M 2024-04-02T14:39:03.314 INFO:teuthology.orchestra.run.smithi183.stdout:Installed size: 123 M 2024-04-02T14:39:03.315 INFO:teuthology.orchestra.run.smithi183.stdout:Downloading Packages: 2024-04-02T14:39:03.578 INFO:teuthology.orchestra.run.smithi183.stdout:(1/4): kernel-5.14.0-434.el9.x86_64.rpm 725 kB/s | 192 kB 00:00 2024-04-02T14:39:04.445 INFO:teuthology.orchestra.run.smithi183.stdout:(2/4): kernel-core-5.14.0-434.el9.x86_64.rpm 13 MB/s | 15 MB 00:01 2024-04-02T14:39:05.171 INFO:teuthology.orchestra.run.smithi183.stdout:(3/4): kernel-modules-core-5.14.0-434.el9.x86_6 17 MB/s | 28 MB 00:01 2024-04-02T14:39:05.546 INFO:teuthology.orchestra.run.smithi183.stdout:(4/4): kernel-modules-5.14.0-434.el9.x86_64.rpm 15 MB/s | 34 MB 00:02 2024-04-02T14:39:05.549 INFO:teuthology.orchestra.run.smithi183.stdout:-------------------------------------------------------------------------------- 2024-04-02T14:39:05.549 INFO:teuthology.orchestra.run.smithi183.stdout:Total 34 MB/s | 76 MB 00:02 2024-04-02T14:39:05.549 INFO:teuthology.orchestra.run.smithi183.stdout:Running transaction check 2024-04-02T14:39:05.736 INFO:teuthology.orchestra.run.smithi122.stdout:lab-extras 14 kB/s | 1.7 kB 00:00 2024-04-02T14:39:05.918 INFO:teuthology.orchestra.run.smithi183.stdout:Transaction check succeeded. 2024-04-02T14:39:05.918 INFO:teuthology.orchestra.run.smithi183.stdout:Running transaction test 2024-04-02T14:39:06.738 INFO:teuthology.orchestra.run.smithi183.stdout:Transaction test succeeded. 2024-04-02T14:39:06.739 INFO:teuthology.orchestra.run.smithi183.stdout:Running transaction 2024-04-02T14:39:07.984 INFO:teuthology.orchestra.run.smithi122.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-02T14:39:07.984 INFO:teuthology.orchestra.run.smithi122.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-02T14:39:08.077 INFO:teuthology.orchestra.run.smithi122.stdout:Dependencies resolved. 2024-04-02T14:39:08.083 INFO:teuthology.orchestra.run.smithi122.stdout:================================================================================ 2024-04-02T14:39:08.083 INFO:teuthology.orchestra.run.smithi122.stdout: Package Arch Version Repository Size 2024-04-02T14:39:08.084 INFO:teuthology.orchestra.run.smithi122.stdout:================================================================================ 2024-04-02T14:39:08.084 INFO:teuthology.orchestra.run.smithi122.stdout:Installing: 2024-04-02T14:39:08.084 INFO:teuthology.orchestra.run.smithi122.stdout: kernel x86_64 5.14.0-434.el9 CentOS-BaseOS 192 k 2024-04-02T14:39:08.084 INFO:teuthology.orchestra.run.smithi122.stdout:Installing dependencies: 2024-04-02T14:39:08.084 INFO:teuthology.orchestra.run.smithi122.stdout: kernel-core x86_64 5.14.0-434.el9 CentOS-BaseOS 15 M 2024-04-02T14:39:08.084 INFO:teuthology.orchestra.run.smithi122.stdout: kernel-modules x86_64 5.14.0-434.el9 CentOS-BaseOS 34 M 2024-04-02T14:39:08.084 INFO:teuthology.orchestra.run.smithi122.stdout: kernel-modules-core x86_64 5.14.0-434.el9 CentOS-BaseOS 28 M 2024-04-02T14:39:08.084 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:39:08.084 INFO:teuthology.orchestra.run.smithi122.stdout:Transaction Summary 2024-04-02T14:39:08.084 INFO:teuthology.orchestra.run.smithi122.stdout:================================================================================ 2024-04-02T14:39:08.084 INFO:teuthology.orchestra.run.smithi122.stdout:Install 4 Packages 2024-04-02T14:39:08.084 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:39:08.085 INFO:teuthology.orchestra.run.smithi122.stdout:Total download size: 76 M 2024-04-02T14:39:08.085 INFO:teuthology.orchestra.run.smithi122.stdout:Installed size: 123 M 2024-04-02T14:39:08.085 INFO:teuthology.orchestra.run.smithi122.stdout:Downloading Packages: 2024-04-02T14:39:08.245 INFO:teuthology.orchestra.run.smithi122.stdout:(1/4): kernel-5.14.0-434.el9.x86_64.rpm 1.2 MB/s | 192 kB 00:00 2024-04-02T14:39:09.004 INFO:teuthology.orchestra.run.smithi122.stdout:(2/4): kernel-core-5.14.0-434.el9.x86_64.rpm 16 MB/s | 15 MB 00:00 2024-04-02T14:39:09.037 INFO:teuthology.orchestra.run.smithi183.stdout: Preparing : 1/1 2024-04-02T14:39:09.620 INFO:teuthology.orchestra.run.smithi183.stdout: Installing : kernel-modules-core-5.14.0-434.el9.x86_64 1/4 2024-04-02T14:39:09.712 INFO:teuthology.orchestra.run.smithi183.stdout: Installing : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T14:39:09.737 INFO:teuthology.orchestra.run.smithi122.stdout:(3/4): kernel-modules-5.14.0-434.el9.x86_64.rpm 20 MB/s | 34 MB 00:01 2024-04-02T14:39:10.196 INFO:teuthology.orchestra.run.smithi122.stdout:(4/4): kernel-modules-core-5.14.0-434.el9.x86_6 14 MB/s | 28 MB 00:01 2024-04-02T14:39:10.199 INFO:teuthology.orchestra.run.smithi122.stdout:-------------------------------------------------------------------------------- 2024-04-02T14:39:10.200 INFO:teuthology.orchestra.run.smithi122.stdout:Total 36 MB/s | 76 MB 00:02 2024-04-02T14:39:10.200 INFO:teuthology.orchestra.run.smithi122.stdout:Running transaction check 2024-04-02T14:39:10.584 INFO:teuthology.orchestra.run.smithi122.stdout:Transaction check succeeded. 2024-04-02T14:39:10.584 INFO:teuthology.orchestra.run.smithi122.stdout:Running transaction test 2024-04-02T14:39:11.430 INFO:teuthology.orchestra.run.smithi122.stdout:Transaction test succeeded. 2024-04-02T14:39:11.431 INFO:teuthology.orchestra.run.smithi122.stdout:Running transaction 2024-04-02T14:39:12.449 INFO:teuthology.orchestra.run.smithi183.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T14:39:12.590 INFO:teuthology.orchestra.run.smithi183.stdout: Installing : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T14:39:13.738 INFO:teuthology.orchestra.run.smithi122.stdout: Preparing : 1/1 2024-04-02T14:39:14.355 INFO:teuthology.orchestra.run.smithi122.stdout: Installing : kernel-modules-core-5.14.0-434.el9.x86_64 1/4 2024-04-02T14:39:14.449 INFO:teuthology.orchestra.run.smithi122.stdout: Installing : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T14:39:17.205 INFO:teuthology.orchestra.run.smithi122.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T14:39:17.351 INFO:teuthology.orchestra.run.smithi183.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T14:39:17.356 INFO:teuthology.orchestra.run.smithi122.stdout: Installing : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T14:39:17.455 INFO:teuthology.orchestra.run.smithi183.stdout: Installing : kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:39:22.051 INFO:teuthology.orchestra.run.smithi122.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T14:39:22.164 INFO:teuthology.orchestra.run.smithi122.stdout: Installing : kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:39:22.187 INFO:teuthology.orchestra.run.smithi183.stdout: Running scriptlet: kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:39:26.876 INFO:teuthology.orchestra.run.smithi122.stdout: Running scriptlet: kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:39:51.610 INFO:teuthology.orchestra.run.smithi183.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:39:51.610 INFO:teuthology.orchestra.run.smithi183.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-02T14:39:51.610 INFO:teuthology.orchestra.run.smithi183.stdout: 2024-04-02T14:39:51.653 INFO:teuthology.orchestra.run.smithi183.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:39:51.994 INFO:teuthology.orchestra.run.smithi183.stdout: Running scriptlet: kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:39:51.994 INFO:teuthology.orchestra.run.smithi183.stdout: Verifying : kernel-5.14.0-434.el9.x86_64 1/4 2024-04-02T14:39:51.994 INFO:teuthology.orchestra.run.smithi183.stdout: Verifying : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T14:39:51.994 INFO:teuthology.orchestra.run.smithi183.stdout: Verifying : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T14:39:52.413 INFO:teuthology.orchestra.run.smithi183.stdout: Verifying : kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:39:52.413 INFO:teuthology.orchestra.run.smithi183.stdout: 2024-04-02T14:39:52.413 INFO:teuthology.orchestra.run.smithi183.stdout:Installed: 2024-04-02T14:39:52.413 INFO:teuthology.orchestra.run.smithi183.stdout: kernel-5.14.0-434.el9.x86_64 2024-04-02T14:39:52.413 INFO:teuthology.orchestra.run.smithi183.stdout: kernel-core-5.14.0-434.el9.x86_64 2024-04-02T14:39:52.413 INFO:teuthology.orchestra.run.smithi183.stdout: kernel-modules-5.14.0-434.el9.x86_64 2024-04-02T14:39:52.413 INFO:teuthology.orchestra.run.smithi183.stdout: kernel-modules-core-5.14.0-434.el9.x86_64 2024-04-02T14:39:52.413 INFO:teuthology.orchestra.run.smithi183.stdout: 2024-04-02T14:39:52.413 INFO:teuthology.orchestra.run.smithi183.stdout:Complete! 2024-04-02T14:39:52.747 DEBUG:teuthology.orchestra.run.smithi183:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T14:39:52.867 INFO:teuthology.orchestra.run.smithi183.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T14:39:52.867 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T14:39:52.868 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-434.el9.x86_64 2024-04-02T14:39:52.868 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-02T14:39:52.868 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-02T14:39:52.868 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-04-02T14:39:52.868 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi183.front.sepia.ceph.com, path=None, version=distro) 2024-04-02T14:39:52.868 DEBUG:teuthology.orchestra.run.smithi183:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T14:39:52.948 INFO:teuthology.orchestra.run.smithi183.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T14:39:52.948 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T14:39:52.948 DEBUG:teuthology.orchestra.run.smithi183:> sudo rpm -qi grub2-tools 2024-04-02T14:39:52.982 INFO:teuthology.orchestra.run.smithi183.stdout:Name : grub2-tools 2024-04-02T14:39:52.983 INFO:teuthology.orchestra.run.smithi183.stdout:Epoch : 1 2024-04-02T14:39:52.983 INFO:teuthology.orchestra.run.smithi183.stdout:Version : 2.06 2024-04-02T14:39:52.983 INFO:teuthology.orchestra.run.smithi183.stdout:Release : 68.el9 2024-04-02T14:39:52.983 INFO:teuthology.orchestra.run.smithi183.stdout:Architecture: x86_64 2024-04-02T14:39:52.983 INFO:teuthology.orchestra.run.smithi183.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-02T14:39:52.983 INFO:teuthology.orchestra.run.smithi183.stdout:Group : Unspecified 2024-04-02T14:39:52.984 INFO:teuthology.orchestra.run.smithi183.stdout:Size : 8264254 2024-04-02T14:39:52.984 INFO:teuthology.orchestra.run.smithi183.stdout:License : GPLv3+ 2024-04-02T14:39:52.984 INFO:teuthology.orchestra.run.smithi183.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-02T14:39:52.984 INFO:teuthology.orchestra.run.smithi183.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-02T14:39:52.984 INFO:teuthology.orchestra.run.smithi183.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-02T14:39:52.984 INFO:teuthology.orchestra.run.smithi183.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-02T14:39:52.985 INFO:teuthology.orchestra.run.smithi183.stdout:Packager : builder@centos.org 2024-04-02T14:39:52.985 INFO:teuthology.orchestra.run.smithi183.stdout:Vendor : CentOS 2024-04-02T14:39:52.985 INFO:teuthology.orchestra.run.smithi183.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-02T14:39:52.985 INFO:teuthology.orchestra.run.smithi183.stdout:Summary : Support tools for GRUB. 2024-04-02T14:39:52.985 INFO:teuthology.orchestra.run.smithi183.stdout:Description : 2024-04-02T14:39:52.985 INFO:teuthology.orchestra.run.smithi183.stdout: 2024-04-02T14:39:52.986 INFO:teuthology.orchestra.run.smithi183.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-02T14:39:52.986 INFO:teuthology.orchestra.run.smithi183.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-02T14:39:52.986 INFO:teuthology.orchestra.run.smithi183.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-02T14:39:52.986 INFO:teuthology.orchestra.run.smithi183.stdout:hardware devices. 2024-04-02T14:39:52.986 INFO:teuthology.orchestra.run.smithi183.stdout: 2024-04-02T14:39:52.986 INFO:teuthology.orchestra.run.smithi183.stdout:This subpackage provides tools for support of all platforms. 2024-04-02T14:39:52.988 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-02T14:39:52.988 INFO:teuthology.task.kernel:Updating grub on smithi183 to boot 5.14.0-434.el9.x86_64 2024-04-02T14:39:52.988 DEBUG:teuthology.orchestra.run.smithi183:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-02T14:39:53.626 INFO:teuthology.orchestra.run.smithi183.stderr:Generating grub configuration file ... 2024-04-02T14:39:55.020 INFO:teuthology.orchestra.run.smithi183.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-02T14:39:55.052 INFO:teuthology.orchestra.run.smithi183.stderr:done 2024-04-02T14:39:55.054 DEBUG:teuthology.orchestra.run.smithi183:> mktemp 2024-04-02T14:39:55.073 INFO:teuthology.orchestra.run.smithi183.stdout:/tmp/tmp.7nhhtZCTkq 2024-04-02T14:39:55.073 DEBUG:teuthology.orchestra.run.smithi183:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.7nhhtZCTkq 2024-04-02T14:39:55.141 DEBUG:teuthology.orchestra.run.smithi183:> sudo chmod 0666 /tmp/tmp.7nhhtZCTkq 2024-04-02T14:39:55.287 DEBUG:teuthology.orchestra.remote:smithi183:/tmp/tmp.7nhhtZCTkq is 6KB 2024-04-02T14:39:55.334 DEBUG:teuthology.orchestra.run.smithi183:> rm -fr /tmp/tmp.7nhhtZCTkq 2024-04-02T14:39:55.348 DEBUG:teuthology.orchestra.run.smithi183:> sudo /bin/ls /boot/loader/entries || true 2024-04-02T14:39:55.418 INFO:teuthology.orchestra.run.smithi183.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-02T14:39:55.419 INFO:teuthology.orchestra.run.smithi183.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-02T14:39:55.419 INFO:teuthology.orchestra.run.smithi183.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-02T14:39:55.419 INFO:teuthology.orchestra.run.smithi183.stdout:15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64.conf 2024-04-02T14:39:55.420 DEBUG:teuthology.orchestra.run.smithi183:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64 2024-04-02T14:39:55.586 DEBUG:teuthology.orchestra.run.smithi183:> sudo shutdown -r now 2024-04-02T14:39:56.219 INFO:teuthology.orchestra.run.smithi122.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:39:56.220 INFO:teuthology.orchestra.run.smithi122.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-02T14:39:56.220 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:39:56.262 INFO:teuthology.orchestra.run.smithi122.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:39:56.702 INFO:teuthology.orchestra.run.smithi122.stdout: Running scriptlet: kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:39:56.703 INFO:teuthology.orchestra.run.smithi122.stdout: Verifying : kernel-5.14.0-434.el9.x86_64 1/4 2024-04-02T14:39:56.703 INFO:teuthology.orchestra.run.smithi122.stdout: Verifying : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T14:39:56.703 INFO:teuthology.orchestra.run.smithi122.stdout: Verifying : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T14:39:57.145 INFO:teuthology.orchestra.run.smithi122.stdout: Verifying : kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T14:39:57.145 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:39:57.145 INFO:teuthology.orchestra.run.smithi122.stdout:Installed: 2024-04-02T14:39:57.145 INFO:teuthology.orchestra.run.smithi122.stdout: kernel-5.14.0-434.el9.x86_64 2024-04-02T14:39:57.145 INFO:teuthology.orchestra.run.smithi122.stdout: kernel-core-5.14.0-434.el9.x86_64 2024-04-02T14:39:57.145 INFO:teuthology.orchestra.run.smithi122.stdout: kernel-modules-5.14.0-434.el9.x86_64 2024-04-02T14:39:57.145 INFO:teuthology.orchestra.run.smithi122.stdout: kernel-modules-core-5.14.0-434.el9.x86_64 2024-04-02T14:39:57.146 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:39:57.146 INFO:teuthology.orchestra.run.smithi122.stdout:Complete! 2024-04-02T14:39:57.473 DEBUG:teuthology.orchestra.run.smithi122:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T14:39:57.575 INFO:teuthology.orchestra.run.smithi122.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T14:39:57.575 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T14:39:57.575 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-434.el9.x86_64 2024-04-02T14:39:57.575 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-02T14:39:57.575 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-02T14:39:57.575 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-04-02T14:39:57.575 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi122.front.sepia.ceph.com, path=None, version=distro) 2024-04-02T14:39:57.575 DEBUG:teuthology.orchestra.run.smithi122:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T14:39:57.655 INFO:teuthology.orchestra.run.smithi122.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T14:39:57.655 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T14:39:57.655 DEBUG:teuthology.orchestra.run.smithi122:> sudo rpm -qi grub2-tools 2024-04-02T14:39:57.689 INFO:teuthology.orchestra.run.smithi122.stdout:Name : grub2-tools 2024-04-02T14:39:57.689 INFO:teuthology.orchestra.run.smithi122.stdout:Epoch : 1 2024-04-02T14:39:57.689 INFO:teuthology.orchestra.run.smithi122.stdout:Version : 2.06 2024-04-02T14:39:57.689 INFO:teuthology.orchestra.run.smithi122.stdout:Release : 68.el9 2024-04-02T14:39:57.689 INFO:teuthology.orchestra.run.smithi122.stdout:Architecture: x86_64 2024-04-02T14:39:57.689 INFO:teuthology.orchestra.run.smithi122.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-02T14:39:57.689 INFO:teuthology.orchestra.run.smithi122.stdout:Group : Unspecified 2024-04-02T14:39:57.689 INFO:teuthology.orchestra.run.smithi122.stdout:Size : 8264254 2024-04-02T14:39:57.689 INFO:teuthology.orchestra.run.smithi122.stdout:License : GPLv3+ 2024-04-02T14:39:57.689 INFO:teuthology.orchestra.run.smithi122.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-02T14:39:57.689 INFO:teuthology.orchestra.run.smithi122.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-02T14:39:57.690 INFO:teuthology.orchestra.run.smithi122.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-02T14:39:57.690 INFO:teuthology.orchestra.run.smithi122.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-02T14:39:57.690 INFO:teuthology.orchestra.run.smithi122.stdout:Packager : builder@centos.org 2024-04-02T14:39:57.690 INFO:teuthology.orchestra.run.smithi122.stdout:Vendor : CentOS 2024-04-02T14:39:57.690 INFO:teuthology.orchestra.run.smithi122.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-02T14:39:57.690 INFO:teuthology.orchestra.run.smithi122.stdout:Summary : Support tools for GRUB. 2024-04-02T14:39:57.690 INFO:teuthology.orchestra.run.smithi122.stdout:Description : 2024-04-02T14:39:57.690 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:39:57.690 INFO:teuthology.orchestra.run.smithi122.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-02T14:39:57.690 INFO:teuthology.orchestra.run.smithi122.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-02T14:39:57.690 INFO:teuthology.orchestra.run.smithi122.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-02T14:39:57.690 INFO:teuthology.orchestra.run.smithi122.stdout:hardware devices. 2024-04-02T14:39:57.690 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:39:57.690 INFO:teuthology.orchestra.run.smithi122.stdout:This subpackage provides tools for support of all platforms. 2024-04-02T14:39:57.691 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-02T14:39:57.691 INFO:teuthology.task.kernel:Updating grub on smithi122 to boot 5.14.0-434.el9.x86_64 2024-04-02T14:39:57.691 DEBUG:teuthology.orchestra.run.smithi122:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-02T14:39:58.317 INFO:teuthology.orchestra.run.smithi122.stderr:Generating grub configuration file ... 2024-04-02T14:39:59.637 INFO:teuthology.orchestra.run.smithi122.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-02T14:39:59.664 INFO:teuthology.orchestra.run.smithi122.stderr:done 2024-04-02T14:39:59.666 DEBUG:teuthology.orchestra.run.smithi122:> mktemp 2024-04-02T14:39:59.681 INFO:teuthology.orchestra.run.smithi122.stdout:/tmp/tmp.DkAteeSj5C 2024-04-02T14:39:59.682 DEBUG:teuthology.orchestra.run.smithi122:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.DkAteeSj5C 2024-04-02T14:39:59.750 DEBUG:teuthology.orchestra.run.smithi122:> sudo chmod 0666 /tmp/tmp.DkAteeSj5C 2024-04-02T14:39:59.895 DEBUG:teuthology.orchestra.remote:smithi122:/tmp/tmp.DkAteeSj5C is 6KB 2024-04-02T14:39:59.943 DEBUG:teuthology.orchestra.run.smithi122:> rm -fr /tmp/tmp.DkAteeSj5C 2024-04-02T14:39:59.956 DEBUG:teuthology.orchestra.run.smithi122:> sudo /bin/ls /boot/loader/entries || true 2024-04-02T14:40:00.021 INFO:teuthology.orchestra.run.smithi122.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-02T14:40:00.021 INFO:teuthology.orchestra.run.smithi122.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-02T14:40:00.021 INFO:teuthology.orchestra.run.smithi122.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-02T14:40:00.021 INFO:teuthology.orchestra.run.smithi122.stdout:15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64.conf 2024-04-02T14:40:00.023 DEBUG:teuthology.orchestra.run.smithi122:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64 2024-04-02T14:40:00.167 DEBUG:teuthology.orchestra.run.smithi122:> sudo shutdown -r now 2024-04-02T14:40:25.613 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-04-02T14:40:25.614 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi183.front.sepia.ceph.com' 2024-04-02T14:40:25.614 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi183.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T14:40:30.174 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-04-02T14:40:30.174 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi122.front.sepia.ceph.com' 2024-04-02T14:40:30.175 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi122.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T14:41:25.670 DEBUG:teuthology.orchestra.remote:timed out 2024-04-02T14:41:30.178 DEBUG:teuthology.orchestra.remote:timed out 2024-04-02T14:41:34.676 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi183.front.sepia.ceph.com' 2024-04-02T14:41:34.677 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi183.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T14:41:38.190 DEBUG:teuthology.orchestra.run.smithi183:> true 2024-04-02T14:41:38.638 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi183.front.sepia.ceph.com' 2024-04-02T14:41:38.638 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-434.el9.x86_64"... 2024-04-02T14:41:38.639 DEBUG:teuthology.orchestra.run.smithi183:> uname -r 2024-04-02T14:41:38.693 INFO:teuthology.orchestra.run.smithi183.stdout:5.14.0-434.el9.x86_64 2024-04-02T14:41:38.693 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-434.el9.x86_64 vs 5.14.0-434.el9.x86_64 2024-04-02T14:41:38.693 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-02T14:41:38.693 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-02T14:41:39.179 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi122.front.sepia.ceph.com' 2024-04-02T14:41:39.181 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi122.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T14:41:39.695 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-04-02T14:41:39.695 DEBUG:teuthology.orchestra.run.smithi183:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-02T14:41:39.773 INFO:teuthology.orchestra.run.smithi183.stdout:ttyS1 2024-04-02T14:41:39.806 DEBUG:teuthology.parallel:result is None 2024-04-02T14:41:42.629 DEBUG:teuthology.orchestra.run.smithi122:> true 2024-04-02T14:41:43.057 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi122.front.sepia.ceph.com' 2024-04-02T14:41:43.057 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-434.el9.x86_64"... 2024-04-02T14:41:43.057 DEBUG:teuthology.orchestra.run.smithi122:> uname -r 2024-04-02T14:41:43.071 INFO:teuthology.orchestra.run.smithi122.stdout:5.14.0-434.el9.x86_64 2024-04-02T14:41:43.072 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-434.el9.x86_64 vs 5.14.0-434.el9.x86_64 2024-04-02T14:41:43.072 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-02T14:41:43.072 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-02T14:41:44.072 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-04-02T14:41:44.073 DEBUG:teuthology.orchestra.run.smithi122:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-02T14:41:44.126 INFO:teuthology.orchestra.run.smithi122.stdout:ttyS1 2024-04-02T14:41:44.181 DEBUG:teuthology.parallel:result is None 2024-04-02T14:41:44.182 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-02T14:41:44.234 INFO:teuthology.task.internal:Creating test directory... 2024-04-02T14:41:44.235 DEBUG:teuthology.orchestra.run.smithi122:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-02T14:41:44.243 DEBUG:teuthology.orchestra.run.smithi183:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-02T14:41:44.273 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-02T14:41:44.418 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-02T14:41:44.489 INFO:teuthology.task.internal:Creating archive directory... 2024-04-02T14:41:44.489 DEBUG:teuthology.orchestra.run.smithi122:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-02T14:41:44.494 DEBUG:teuthology.orchestra.run.smithi183:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-02T14:41:44.713 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-02T14:41:45.140 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-02T14:41:45.140 DEBUG:teuthology.orchestra.run.smithi122:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-02T14:41:45.143 DEBUG:teuthology.orchestra.run.smithi183:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-02T14:41:45.172 INFO:teuthology.orchestra.run.smithi183.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T14:41:45.174 INFO:teuthology.orchestra.run.smithi122.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T14:41:45.185 INFO:teuthology.orchestra.run.smithi183.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T14:41:45.186 INFO:teuthology.orchestra.run.smithi122.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T14:41:45.188 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-02T14:41:45.208 INFO:teuthology.task.internal:Configuring sudo... 2024-04-02T14:41:45.209 DEBUG:teuthology.orchestra.run.smithi122:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-02T14:41:45.231 DEBUG:teuthology.orchestra.run.smithi183:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-02T14:41:45.258 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-02T14:41:45.308 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-02T14:41:45.309 DEBUG:teuthology.orchestra.run.smithi122:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-02T14:41:45.311 DEBUG:teuthology.orchestra.run.smithi183:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-02T14:41:45.328 DEBUG:teuthology.orchestra.run.smithi122:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T14:41:45.402 DEBUG:teuthology.orchestra.run.smithi122:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T14:41:45.456 DEBUG:teuthology.orchestra.run.smithi122:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T14:41:45.540 DEBUG:teuthology.orchestra.run.smithi122:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T14:41:45.584 DEBUG:teuthology.orchestra.run.smithi122:> set -ex 2024-04-02T14:41:45.584 DEBUG:teuthology.orchestra.run.smithi122:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-02T14:41:45.655 DEBUG:teuthology.orchestra.run.smithi183:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T14:41:45.699 DEBUG:teuthology.orchestra.run.smithi183:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T14:41:45.757 DEBUG:teuthology.orchestra.run.smithi183:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T14:41:45.841 DEBUG:teuthology.orchestra.run.smithi183:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T14:41:45.886 DEBUG:teuthology.orchestra.run.smithi183:> set -ex 2024-04-02T14:41:45.886 DEBUG:teuthology.orchestra.run.smithi183:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-02T14:41:45.952 DEBUG:teuthology.orchestra.run.smithi122:> sudo service rsyslog restart 2024-04-02T14:41:45.954 DEBUG:teuthology.orchestra.run.smithi183:> sudo service rsyslog restart 2024-04-02T14:41:46.000 INFO:teuthology.orchestra.run.smithi122.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-02T14:41:46.040 INFO:teuthology.orchestra.run.smithi183.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-02T14:41:46.548 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-02T14:41:46.556 INFO:teuthology.task.internal:Starting timer... 2024-04-02T14:41:46.556 INFO:teuthology.run_tasks:Running task pcp... 2024-04-02T14:41:46.593 INFO:teuthology.run_tasks:Running task selinux... 2024-04-02T14:41:46.641 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-04-02T14:41:46.641 DEBUG:teuthology.orchestra.run.smithi122:> sudo service auditd rotate 2024-04-02T14:41:46.695 INFO:teuthology.orchestra.run.smithi122.stdout:Rotating logs: 2024-04-02T14:41:46.696 DEBUG:teuthology.orchestra.run.smithi183:> sudo service auditd rotate 2024-04-02T14:41:46.766 INFO:teuthology.orchestra.run.smithi183.stdout:Rotating logs: 2024-04-02T14:41:46.768 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-02T14:41:46.768 DEBUG:teuthology.orchestra.run.smithi122:> /usr/sbin/getenforce 2024-04-02T14:41:46.789 INFO:teuthology.orchestra.run.smithi122.stdout:Permissive 2024-04-02T14:41:46.789 DEBUG:teuthology.orchestra.run.smithi183:> /usr/sbin/getenforce 2024-04-02T14:41:46.816 INFO:teuthology.orchestra.run.smithi183.stdout:Permissive 2024-04-02T14:41:46.817 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi122.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi183.front.sepia.ceph.com': 'permissive'} 2024-04-02T14:41:46.817 DEBUG:teuthology.orchestra.run.smithi122:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-02T14:41:46.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T14:41:46.861 DEBUG:teuthology.orchestra.run.smithi183:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-02T14:41:46.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T14:41:46.889 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-02T14:41:46.889 DEBUG:teuthology.orchestra.run.smithi122:> sudo /usr/sbin/setenforce permissive 2024-04-02T14:41:47.042 DEBUG:teuthology.orchestra.run.smithi183:> sudo /usr/sbin/setenforce permissive 2024-04-02T14:41:47.077 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-02T14:41:47.087 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-02T14:41:47.092 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-02T14:41:47.232 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-02T14:41:47.249 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-02T14:41:47.250 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi122.front.sepia.ceph.com,smithi183.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-02T14:46:05.641 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi122.front.sepia.ceph.com'), Remote(name='ubuntu@smithi183.front.sepia.ceph.com')] 2024-04-02T14:46:05.642 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi122.front.sepia.ceph.com' 2024-04-02T14:46:05.643 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi122.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T14:46:05.739 DEBUG:teuthology.orchestra.run.smithi122:> true 2024-04-02T14:46:05.816 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi122.front.sepia.ceph.com' 2024-04-02T14:46:05.817 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi183.front.sepia.ceph.com' 2024-04-02T14:46:05.817 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi183.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T14:46:05.904 DEBUG:teuthology.orchestra.run.smithi183:> true 2024-04-02T14:46:05.985 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi183.front.sepia.ceph.com' 2024-04-02T14:46:05.986 INFO:teuthology.run_tasks:Running task clock... 2024-04-02T14:46:05.996 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-02T14:46:05.996 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-02T14:46:05.996 DEBUG:teuthology.orchestra.run.smithi122:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-02T14:46:05.999 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-02T14:46:05.999 DEBUG:teuthology.orchestra.run.smithi183:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-02T14:46:06.027 INFO:teuthology.orchestra.run.smithi122.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-02T14:46:06.043 INFO:teuthology.orchestra.run.smithi122.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-02T14:46:06.057 INFO:teuthology.orchestra.run.smithi183.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-02T14:46:06.073 INFO:teuthology.orchestra.run.smithi122.stderr:sudo: ntpd: command not found 2024-04-02T14:46:06.074 INFO:teuthology.orchestra.run.smithi183.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-02T14:46:06.086 INFO:teuthology.orchestra.run.smithi122.stdout:506 Cannot talk to daemon 2024-04-02T14:46:06.103 INFO:teuthology.orchestra.run.smithi183.stderr:sudo: ntpd: command not found 2024-04-02T14:46:06.104 INFO:teuthology.orchestra.run.smithi122.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-02T14:46:06.117 INFO:teuthology.orchestra.run.smithi183.stdout:506 Cannot talk to daemon 2024-04-02T14:46:06.120 INFO:teuthology.orchestra.run.smithi122.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-02T14:46:06.134 INFO:teuthology.orchestra.run.smithi183.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-02T14:46:06.150 INFO:teuthology.orchestra.run.smithi183.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-02T14:46:06.178 INFO:teuthology.orchestra.run.smithi122.stderr:bash: line 1: ntpq: command not found 2024-04-02T14:46:06.181 INFO:teuthology.orchestra.run.smithi122.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-02T14:46:06.181 INFO:teuthology.orchestra.run.smithi122.stdout:=============================================================================== 2024-04-02T14:46:06.181 INFO:teuthology.orchestra.run.smithi122.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T14:46:06.181 INFO:teuthology.orchestra.run.smithi122.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T14:46:06.181 INFO:teuthology.orchestra.run.smithi122.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T14:46:06.182 INFO:teuthology.orchestra.run.smithi122.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T14:46:06.205 INFO:teuthology.orchestra.run.smithi183.stderr:bash: line 1: ntpq: command not found 2024-04-02T14:46:06.208 INFO:teuthology.orchestra.run.smithi183.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-02T14:46:06.208 INFO:teuthology.orchestra.run.smithi183.stdout:=============================================================================== 2024-04-02T14:46:06.208 INFO:teuthology.orchestra.run.smithi183.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T14:46:06.208 INFO:teuthology.orchestra.run.smithi183.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T14:46:06.208 INFO:teuthology.orchestra.run.smithi183.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T14:46:06.208 INFO:teuthology.orchestra.run.smithi183.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T14:46:06.209 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-02T14:46:06.306 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '3f3f3ed992429b85be729743ad3b39d52dd90744'} 2024-04-02T14:46:06.306 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-04-02T14:46:06.307 INFO:tasks.cephadm:Cluster fsid is bc643104-f0ff-11ee-b647-cb9ed24678a4 2024-04-02T14:46:06.307 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-02T14:46:06.307 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-04-02T14:46:06.307 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi122': '172.21.15.122', 'mon.smithi183': '172.21.15.183'} 2024-04-02T14:46:06.307 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-02T14:46:06.308 DEBUG:teuthology.orchestra.run.smithi122:> sudo hostname $(hostname -s) 2024-04-02T14:46:06.336 DEBUG:teuthology.orchestra.run.smithi183:> sudo hostname $(hostname -s) 2024-04-02T14:46:06.364 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-04-02T14:46:06.364 DEBUG:teuthology.orchestra.run.smithi122:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-02T14:46:06.656 INFO:teuthology.orchestra.run.smithi122.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 2 14:46 /home/ubuntu/cephtest/cephadm 2024-04-02T14:46:06.657 DEBUG:teuthology.orchestra.run.smithi183:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-02T14:46:06.830 INFO:teuthology.orchestra.run.smithi183.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 2 14:46 /home/ubuntu/cephtest/cephadm 2024-04-02T14:46:06.830 DEBUG:teuthology.orchestra.run.smithi122:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-02T14:46:06.846 DEBUG:teuthology.orchestra.run.smithi183:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-02T14:46:06.873 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-04-02T14:46:06.873 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-02T14:46:06.889 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-02T14:46:07.783 INFO:teuthology.orchestra.run.smithi122.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-02T14:46:07.829 INFO:teuthology.orchestra.run.smithi183.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-02T14:46:29.665 INFO:teuthology.orchestra.run.smithi122.stdout:{ 2024-04-02T14:46:29.665 INFO:teuthology.orchestra.run.smithi122.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-02T14:46:29.665 INFO:teuthology.orchestra.run.smithi122.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-02T14:46:29.665 INFO:teuthology.orchestra.run.smithi122.stdout: "repo_digests": [ 2024-04-02T14:46:29.665 INFO:teuthology.orchestra.run.smithi122.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-02T14:46:29.665 INFO:teuthology.orchestra.run.smithi122.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-02T14:46:29.665 INFO:teuthology.orchestra.run.smithi122.stdout: ] 2024-04-02T14:46:29.665 INFO:teuthology.orchestra.run.smithi122.stdout:} 2024-04-02T14:46:30.924 INFO:teuthology.orchestra.run.smithi183.stdout:{ 2024-04-02T14:46:30.925 INFO:teuthology.orchestra.run.smithi183.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-02T14:46:30.925 INFO:teuthology.orchestra.run.smithi183.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-02T14:46:30.925 INFO:teuthology.orchestra.run.smithi183.stdout: "repo_digests": [ 2024-04-02T14:46:30.925 INFO:teuthology.orchestra.run.smithi183.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-02T14:46:30.925 INFO:teuthology.orchestra.run.smithi183.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-02T14:46:30.925 INFO:teuthology.orchestra.run.smithi183.stdout: ] 2024-04-02T14:46:30.925 INFO:teuthology.orchestra.run.smithi183.stdout:} 2024-04-02T14:46:30.943 DEBUG:teuthology.orchestra.run.smithi122:> sudo mkdir -p /etc/ceph 2024-04-02T14:46:30.978 DEBUG:teuthology.orchestra.run.smithi183:> sudo mkdir -p /etc/ceph 2024-04-02T14:46:31.007 DEBUG:teuthology.orchestra.run.smithi122:> sudo chmod 777 /etc/ceph 2024-04-02T14:46:31.051 DEBUG:teuthology.orchestra.run.smithi183:> sudo chmod 777 /etc/ceph 2024-04-02T14:46:31.077 INFO:tasks.cephadm:Writing seed config... 2024-04-02T14:46:31.078 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-02T14:46:31.079 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-02T14:46:31.079 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-02T14:46:31.079 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-02T14:46:31.079 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-02T14:46:31.079 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-02T14:46:31.079 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-02T14:46:31.079 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-02T14:46:31.079 DEBUG:teuthology.orchestra.run.smithi122:> set -ex 2024-04-02T14:46:31.080 DEBUG:teuthology.orchestra.run.smithi122:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-02T14:46:31.108 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = bc643104-f0ff-11ee-b647-cb9ed24678a4 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-02T14:46:31.109 DEBUG:teuthology.orchestra.run.smithi122:mon.smithi122> sudo journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@mon.smithi122.service 2024-04-02T14:46:31.151 INFO:tasks.cephadm:Bootstrapping... 2024-04-02T14:46:31.151 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.122 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-02T14:46:31.349 INFO:teuthology.orchestra.run.smithi122.stderr:-------------------------------------------------------------------------------- 2024-04-02T14:46:31.349 INFO:teuthology.orchestra.run.smithi122.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'bc643104-f0ff-11ee-b647-cb9ed24678a4', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.122', '--skip-admin-label'] 2024-04-02T14:46:31.373 INFO:teuthology.orchestra.run.smithi122.stderr:/bin/podman: 4.9.4-dev 2024-04-02T14:46:31.375 INFO:teuthology.orchestra.run.smithi122.stderr:Verifying podman|docker is present... 2024-04-02T14:46:31.396 INFO:teuthology.orchestra.run.smithi122.stderr:/bin/podman: 4.9.4-dev 2024-04-02T14:46:31.398 INFO:teuthology.orchestra.run.smithi122.stderr:Verifying lvm2 is present... 2024-04-02T14:46:31.398 INFO:teuthology.orchestra.run.smithi122.stderr:Verifying time synchronization is in place... 2024-04-02T14:46:31.405 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-02T14:46:31.411 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: inactive 2024-04-02T14:46:31.417 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: enabled 2024-04-02T14:46:31.426 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: active 2024-04-02T14:46:31.427 INFO:teuthology.orchestra.run.smithi122.stderr:Unit chronyd.service is enabled and running 2024-04-02T14:46:31.427 INFO:teuthology.orchestra.run.smithi122.stderr:Repeating the final host check... 2024-04-02T14:46:31.444 INFO:teuthology.orchestra.run.smithi122.stderr:/bin/podman: 4.9.4-dev 2024-04-02T14:46:31.447 INFO:teuthology.orchestra.run.smithi122.stderr:podman (/bin/podman) version 4.9.4 is present 2024-04-02T14:46:31.447 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl is present 2024-04-02T14:46:31.447 INFO:teuthology.orchestra.run.smithi122.stderr:lvcreate is present 2024-04-02T14:46:31.453 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-02T14:46:31.459 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: inactive 2024-04-02T14:46:31.465 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: enabled 2024-04-02T14:46:31.471 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: active 2024-04-02T14:46:31.472 INFO:teuthology.orchestra.run.smithi122.stderr:Unit chronyd.service is enabled and running 2024-04-02T14:46:31.472 INFO:teuthology.orchestra.run.smithi122.stderr:Host looks OK 2024-04-02T14:46:31.472 INFO:teuthology.orchestra.run.smithi122.stderr:Cluster fsid: bc643104-f0ff-11ee-b647-cb9ed24678a4 2024-04-02T14:46:31.472 INFO:teuthology.orchestra.run.smithi122.stderr:Acquiring lock 139996552568352 on /run/cephadm/bc643104-f0ff-11ee-b647-cb9ed24678a4.lock 2024-04-02T14:46:31.472 INFO:teuthology.orchestra.run.smithi122.stderr:Lock 139996552568352 acquired on /run/cephadm/bc643104-f0ff-11ee-b647-cb9ed24678a4.lock 2024-04-02T14:46:31.472 INFO:teuthology.orchestra.run.smithi122.stderr:Verifying IP 172.21.15.122 port 3300 ... 2024-04-02T14:46:31.473 INFO:teuthology.orchestra.run.smithi122.stderr:Verifying IP 172.21.15.122 port 6789 ... 2024-04-02T14:46:31.473 INFO:teuthology.orchestra.run.smithi122.stderr:Base mon IP is 172.21.15.122, final addrv is [v2:172.21.15.122:3300,v1:172.21.15.122:6789] 2024-04-02T14:46:31.475 INFO:teuthology.orchestra.run.smithi122.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.122 metric 100 2024-04-02T14:46:31.475 INFO:teuthology.orchestra.run.smithi122.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.122 metric 100 2024-04-02T14:46:31.478 INFO:teuthology.orchestra.run.smithi122.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-02T14:46:31.478 INFO:teuthology.orchestra.run.smithi122.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-04-02T14:46:31.478 INFO:teuthology.orchestra.run.smithi122.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1789sec hoplimit 64 pref medium 2024-04-02T14:46:31.480 INFO:teuthology.orchestra.run.smithi122.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-02T14:46:31.480 INFO:teuthology.orchestra.run.smithi122.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-02T14:46:31.480 INFO:teuthology.orchestra.run.smithi122.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-02T14:46:31.481 INFO:teuthology.orchestra.run.smithi122.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-02T14:46:31.481 INFO:teuthology.orchestra.run.smithi122.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d515/64 scope link 2024-04-02T14:46:31.481 INFO:teuthology.orchestra.run.smithi122.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-02T14:46:31.481 INFO:teuthology.orchestra.run.smithi122.stderr:Mon IP `172.21.15.122` is in CIDR network `172.21.0.0/20` 2024-04-02T14:46:31.481 INFO:teuthology.orchestra.run.smithi122.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-02T14:46:31.482 INFO:teuthology.orchestra.run.smithi122.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-02T14:46:31.581 INFO:teuthology.orchestra.run.smithi122.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-04-02T14:46:32.224 INFO:teuthology.orchestra.run.smithi122.stderr:/bin/podman: Getting image source signatures 2024-04-02T14:46:32.224 INFO:teuthology.orchestra.run.smithi122.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-04-02T14:46:32.224 INFO:teuthology.orchestra.run.smithi122.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-04-02T14:46:32.225 INFO:teuthology.orchestra.run.smithi122.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-04-02T14:46:32.225 INFO:teuthology.orchestra.run.smithi122.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-04-02T14:46:32.225 INFO:teuthology.orchestra.run.smithi122.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-04-02T14:46:32.225 INFO:teuthology.orchestra.run.smithi122.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-02T14:46:32.344 INFO:teuthology.orchestra.run.smithi122.stderr:/bin/podman: Writing manifest to image destination 2024-04-02T14:46:32.447 INFO:teuthology.orchestra.run.smithi122.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-02T14:46:32.840 INFO:teuthology.orchestra.run.smithi122.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-02T14:46:33.345 INFO:teuthology.orchestra.run.smithi122.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-02T14:46:33.345 INFO:teuthology.orchestra.run.smithi122.stderr:Extracting ceph user uid/gid from container image... 2024-04-02T14:46:33.704 INFO:teuthology.orchestra.run.smithi122.stderr:stat: 167 167 2024-04-02T14:46:34.160 INFO:teuthology.orchestra.run.smithi122.stderr:Creating initial keys... 2024-04-02T14:46:34.606 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph-authtool: AQBKGgxmHAOdHRAAkfxcxeEMzScS09PUm2jrxw== 2024-04-02T14:46:35.506 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph-authtool: AQBLGgxmS6R8GRAAkv5TV61JH3NqrLIUJBx+FA== 2024-04-02T14:46:36.531 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph-authtool: AQBMGgxm3atiGxAAUq5PNeDw5JBn2/OkR6lIeA== 2024-04-02T14:46:36.977 INFO:teuthology.orchestra.run.smithi122.stderr:Creating initial monmap... 2024-04-02T14:46:37.395 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-02T14:46:37.438 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-04-02T14:46:37.438 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to bc643104-f0ff-11ee-b647-cb9ed24678a4 2024-04-02T14:46:37.438 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-02T14:46:37.916 INFO:teuthology.orchestra.run.smithi122.stderr:monmaptool for smithi122 [v2:172.21.15.122:3300,v1:172.21.15.122:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-02T14:46:37.916 INFO:teuthology.orchestra.run.smithi122.stderr:setting min_mon_release = octopus 2024-04-02T14:46:37.916 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/monmaptool: set fsid to bc643104-f0ff-11ee-b647-cb9ed24678a4 2024-04-02T14:46:37.916 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-02T14:46:37.916 INFO:teuthology.orchestra.run.smithi122.stderr: 2024-04-02T14:46:37.916 INFO:teuthology.orchestra.run.smithi122.stderr:Creating mon... 2024-04-02T14:46:38.856 INFO:teuthology.orchestra.run.smithi122.stderr:create mon.smithi122 on 2024-04-02T14:46:39.064 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-02T14:46:39.212 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4.target → /etc/systemd/system/ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4.target. 2024-04-02T14:46:39.212 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4.target → /etc/systemd/system/ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4.target. 2024-04-02T14:46:39.510 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: Failed to reset failed state of unit ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@mon.smithi122.service: Unit ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@mon.smithi122.service not loaded. 2024-04-02T14:46:39.517 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: Created symlink /etc/systemd/system/ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4.target.wants/ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@mon.smithi122.service → /etc/systemd/system/ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@.service. 2024-04-02T14:46:40.161 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: disabled 2024-04-02T14:46:40.167 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: inactive 2024-04-02T14:46:40.168 INFO:teuthology.orchestra.run.smithi122.stderr:firewalld.service is not enabled 2024-04-02T14:46:40.168 INFO:teuthology.orchestra.run.smithi122.stderr:Not possible to enable service . firewalld.service is not available 2024-04-02T14:46:40.168 INFO:teuthology.orchestra.run.smithi122.stderr:Waiting for mon to start... 2024-04-02T14:46:40.168 INFO:teuthology.orchestra.run.smithi122.stderr:Waiting for mon... 2024-04-02T14:46:40.274 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:40 smithi122 systemd[1]: Started Ceph mon.smithi122 for bc643104-f0ff-11ee-b647-cb9ed24678a4. 2024-04-02T14:46:40.679 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: cluster: 2024-04-02T14:46:40.679 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: id: bc643104-f0ff-11ee-b647-cb9ed24678a4 2024-04-02T14:46:40.679 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-02T14:46:40.679 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: 2024-04-02T14:46:40.679 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: services: 2024-04-02T14:46:40.679 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi122 (age 0.548921s) 2024-04-02T14:46:40.679 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-02T14:46:40.679 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-02T14:46:40.680 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: 2024-04-02T14:46:40.680 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: data: 2024-04-02T14:46:40.680 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-02T14:46:40.680 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-02T14:46:40.680 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-02T14:46:40.680 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: pgs: 2024-04-02T14:46:40.680 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: 2024-04-02T14:46:41.228 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:41 smithi122 ceph-mon[24457]: mon.smithi122 is new leader, mons smithi122 in quorum (ranks 0) 2024-04-02T14:46:41.229 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:41 smithi122 ceph-mon[24457]: monmap e1: 1 mons at {smithi122=[v2:172.21.15.122:3300/0,v1:172.21.15.122:6789/0]} 2024-04-02T14:46:41.229 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:41 smithi122 ceph-mon[24457]: fsmap 2024-04-02T14:46:41.229 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:41 smithi122 ceph-mon[24457]: osdmap e1: 0 total, 0 up, 0 in 2024-04-02T14:46:41.229 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:41 smithi122 ceph-mon[24457]: mgrmap e1: no daemons active 2024-04-02T14:46:41.229 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:41 smithi122 ceph-mon[24457]: from='client.? 172.21.15.122:0/2216997569' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T14:46:41.231 INFO:teuthology.orchestra.run.smithi122.stderr:mon is available 2024-04-02T14:46:41.231 INFO:teuthology.orchestra.run.smithi122.stderr:Assimilating anything we can from ceph.conf... 2024-04-02T14:46:41.721 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: 2024-04-02T14:46:41.722 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: [global] 2024-04-02T14:46:41.722 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: fsid = bc643104-f0ff-11ee-b647-cb9ed24678a4 2024-04-02T14:46:41.722 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.122:3300,v1:172.21.15.122:6789] 2024-04-02T14:46:41.722 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-02T14:46:41.722 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-02T14:46:41.722 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-02T14:46:41.722 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-02T14:46:41.722 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: 2024-04-02T14:46:41.722 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: [mgr] 2024-04-02T14:46:41.722 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-02T14:46:41.722 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: 2024-04-02T14:46:41.722 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: [osd] 2024-04-02T14:46:41.722 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-02T14:46:41.723 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-02T14:46:42.237 INFO:teuthology.orchestra.run.smithi122.stderr:Generating new minimal ceph.conf... 2024-04-02T14:46:42.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:42 smithi122 ceph-mon[24457]: from='client.? 172.21.15.122:0/3559494035' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-02T14:46:42.342 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:42 smithi122 ceph-mon[24457]: from='client.? 172.21.15.122:0/3559494035' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-02T14:46:43.235 INFO:teuthology.orchestra.run.smithi122.stderr:Restarting the monitor... 2024-04-02T14:46:43.287 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:43 smithi122 ceph-mon[24457]: from='client.? 172.21.15.122:0/525600107' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:46:43.287 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:43 smithi122 systemd[1]: Stopping Ceph mon.smithi122 for bc643104-f0ff-11ee-b647-cb9ed24678a4... 2024-04-02T14:46:43.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:43 smithi122 bash[24673]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon.smithi122" found: no such container 2024-04-02T14:46:43.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:43 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi122[24453]: 2024-04-02T14:46:43.398+0000 7f2922ccf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi122 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T14:46:43.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:43 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi122[24453]: 2024-04-02T14:46:43.398+0000 7f2922ccf700 -1 mon.smithi122@0(leader) e1 *** Got Signal Terminated *** 2024-04-02T14:46:43.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:43 smithi122 podman[24685]: 2024-04-02 14:46:43.420387256 +0000 UTC m=+0.119401721 container died ae6ea61ce2d2aa6c0a835ad2499b144145258675fb4b2e088f4d80fedf67785d (image=quay.io/ceph/ceph:v17.2.0, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi122, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, ceph=True, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, version=8, CEPH_POINT_RELEASE=-17.2.0) 2024-04-02T14:46:44.036 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:43 smithi122 podman[24685]: 2024-04-02 14:46:43.774364268 +0000 UTC m=+0.473378741 container cleanup ae6ea61ce2d2aa6c0a835ad2499b144145258675fb4b2e088f4d80fedf67785d (image=quay.io/ceph/ceph:v17.2.0, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi122, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, maintainer=Guillaume Abrioux , vcs-type=git, release=754, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, name=centos-stream, distribution-scope=public) 2024-04-02T14:46:44.036 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:43 smithi122 bash[24685]: ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi122 2024-04-02T14:46:44.036 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:43 smithi122 bash[24713]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon.smithi122" found: no such container 2024-04-02T14:46:44.342 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 podman[24700]: 2024-04-02 14:46:44.041716045 +0000 UTC m=+0.621397253 container remove ae6ea61ce2d2aa6c0a835ad2499b144145258675fb4b2e088f4d80fedf67785d (image=quay.io/ceph/ceph:v17.2.0, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi122, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, vcs-type=git, io.openshift.expose-services=, release=754, GIT_CLEAN=True, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc.) 2024-04-02T14:46:44.342 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 systemd[1]: ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@mon.smithi122.service: Deactivated successfully. 2024-04-02T14:46:44.342 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 systemd[1]: Stopped Ceph mon.smithi122 for bc643104-f0ff-11ee-b647-cb9ed24678a4. 2024-04-02T14:46:44.342 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 systemd[1]: Starting Ceph mon.smithi122 for bc643104-f0ff-11ee-b647-cb9ed24678a4... 2024-04-02T14:46:44.342 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 podman[24783]: 2024-04-02 14:46:44.259237948 +0000 UTC m=+0.107539529 container create d8f91f766e220cb85c34a154aa046e9156ff779b665feb00ddfedd960176f3e9 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi122, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, distribution-scope=public, vendor=Red Hat, Inc., GIT_CLEAN=True, version=8, release=754, vcs-type=git, com.redhat.component=centos-stream-container) 2024-04-02T14:46:44.342 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 podman[24783]: 2024-04-02 14:46:44.170193611 +0000 UTC m=+0.018495207 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-02T14:46:44.509 INFO:teuthology.orchestra.run.smithi122.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-02T14:46:44.624 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 podman[24783]: 2024-04-02 14:46:44.453547587 +0000 UTC m=+0.301849159 container init d8f91f766e220cb85c34a154aa046e9156ff779b665feb00ddfedd960176f3e9 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi122, version=8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., release=754, GIT_CLEAN=True, RELEASE=HEAD, architecture=x86_64, name=centos-stream, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870) 2024-04-02T14:46:44.624 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 podman[24783]: 2024-04-02 14:46:44.456310458 +0000 UTC m=+0.304612031 container start d8f91f766e220cb85c34a154aa046e9156ff779b665feb00ddfedd960176f3e9 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi122, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.openshift.expose-services=, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, architecture=x86_64, maintainer=Guillaume Abrioux , distribution-scope=public, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, com.redhat.component=centos-stream-container, version=8, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD) 2024-04-02T14:46:44.624 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: set uid:gid to 167:167 (ceph:ceph) 2024-04-02T14:46:44.624 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-02T14:46:44.624 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: pidfile_write: ignore empty --pid-file 2024-04-02T14:46:44.624 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: load: jerasure load: lrc 2024-04-02T14:46:44.624 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: RocksDB version: 6.15.5 2024-04-02T14:46:44.625 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-02T14:46:44.625 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Compile date Apr 18 2022 2024-04-02T14:46:44.625 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: DB SUMMARY 2024-04-02T14:46:44.625 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: DB Session ID: V4BPC665WK85PWDFIKQS 2024-04-02T14:46:44.625 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: CURRENT file: CURRENT 2024-04-02T14:46:44.625 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: IDENTITY file: IDENTITY 2024-04-02T14:46:44.625 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-02T14:46:44.625 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi122/store.db dir, Total Num: 1, files: 000008.sst 2024-04-02T14:46:44.625 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi122/store.db: 000010.log size: 82739 ; 2024-04-02T14:46:44.625 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.error_if_exists: 0 2024-04-02T14:46:44.625 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.create_if_missing: 0 2024-04-02T14:46:44.625 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.paranoid_checks: 1 2024-04-02T14:46:44.625 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-02T14:46:44.626 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.env: 0x55c8fb6c6860 2024-04-02T14:46:44.626 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.fs: Posix File System 2024-04-02T14:46:44.626 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.info_log: 0x55c8fcf6dfa0 2024-04-02T14:46:44.626 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_file_opening_threads: 16 2024-04-02T14:46:44.626 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.statistics: (nil) 2024-04-02T14:46:44.626 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.use_fsync: 0 2024-04-02T14:46:44.626 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_log_file_size: 0 2024-04-02T14:46:44.626 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-02T14:46:44.626 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-02T14:46:44.626 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.keep_log_file_num: 1000 2024-04-02T14:46:44.626 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.recycle_log_file_num: 0 2024-04-02T14:46:44.626 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.allow_fallocate: 1 2024-04-02T14:46:44.626 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.allow_mmap_reads: 0 2024-04-02T14:46:44.626 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.allow_mmap_writes: 0 2024-04-02T14:46:44.626 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.use_direct_reads: 0 2024-04-02T14:46:44.627 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-02T14:46:44.627 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.create_missing_column_families: 0 2024-04-02T14:46:44.627 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.db_log_dir: 2024-04-02T14:46:44.627 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi122/store.db 2024-04-02T14:46:44.627 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-02T14:46:44.627 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-02T14:46:44.627 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-02T14:46:44.627 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-02T14:46:44.627 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-02T14:46:44.627 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-02T14:46:44.627 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.advise_random_on_open: 1 2024-04-02T14:46:44.627 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.db_write_buffer_size: 0 2024-04-02T14:46:44.627 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.write_buffer_manager: 0x55c8fd05e2d0 2024-04-02T14:46:44.627 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-02T14:46:44.627 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-02T14:46:44.627 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-02T14:46:44.628 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-02T14:46:44.628 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.rate_limiter: (nil) 2024-04-02T14:46:44.628 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-02T14:46:44.628 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.wal_recovery_mode: 2 2024-04-02T14:46:44.628 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.enable_thread_tracking: 0 2024-04-02T14:46:44.628 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.enable_pipelined_write: 0 2024-04-02T14:46:44.628 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.unordered_write: 0 2024-04-02T14:46:44.628 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-02T14:46:44.628 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-02T14:46:44.628 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-02T14:46:44.628 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-02T14:46:44.628 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.row_cache: None 2024-04-02T14:46:44.628 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.wal_filter: None 2024-04-02T14:46:44.628 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-02T14:46:44.628 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.allow_ingest_behind: 0 2024-04-02T14:46:44.628 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.preserve_deletes: 0 2024-04-02T14:46:44.628 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.two_write_queues: 0 2024-04-02T14:46:44.629 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.manual_wal_flush: 0 2024-04-02T14:46:44.629 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.atomic_flush: 0 2024-04-02T14:46:44.629 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-02T14:46:44.629 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-02T14:46:44.629 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-02T14:46:44.629 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.log_readahead_size: 0 2024-04-02T14:46:44.629 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-02T14:46:44.629 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.best_efforts_recovery: 0 2024-04-02T14:46:44.629 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-02T14:46:44.629 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-02T14:46:44.629 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.allow_data_in_errors: 0 2024-04-02T14:46:44.629 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.db_host_id: __hostname__ 2024-04-02T14:46:44.629 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_background_jobs: 2 2024-04-02T14:46:44.629 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_background_compactions: -1 2024-04-02T14:46:44.629 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_subcompactions: 1 2024-04-02T14:46:44.629 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-02T14:46:44.630 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-02T14:46:44.630 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-02T14:46:44.631 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_total_wal_size: 0 2024-04-02T14:46:44.631 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-02T14:46:44.631 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-02T14:46:44.631 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-02T14:46:44.631 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-02T14:46:44.631 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_open_files: -1 2024-04-02T14:46:44.631 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.bytes_per_sync: 0 2024-04-02T14:46:44.631 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-02T14:46:44.631 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-02T14:46:44.631 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compaction_readahead_size: 0 2024-04-02T14:46:44.631 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_background_flushes: -1 2024-04-02T14:46:44.631 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Compression algorithms supported: 2024-04-02T14:46:44.632 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-02T14:46:44.632 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: kZSTD supported: 0 2024-04-02T14:46:44.632 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: kXpressCompression supported: 0 2024-04-02T14:46:44.632 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: kLZ4HCCompression supported: 1 2024-04-02T14:46:44.632 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: kLZ4Compression supported: 1 2024-04-02T14:46:44.632 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: kBZip2Compression supported: 0 2024-04-02T14:46:44.632 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: kZlibCompression supported: 1 2024-04-02T14:46:44.632 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: kSnappyCompression supported: 1 2024-04-02T14:46:44.632 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-02T14:46:44.632 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi122/store.db/MANIFEST-000009 2024-04-02T14:46:44.632 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-02T14:46:44.632 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-02T14:46:44.632 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.merge_operator: 2024-04-02T14:46:44.632 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compaction_filter: None 2024-04-02T14:46:44.632 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compaction_filter_factory: None 2024-04-02T14:46:44.632 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.sst_partitioner_factory: None 2024-04-02T14:46:44.632 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-02T14:46:44.633 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-02T14:46:44.633 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c8fcf39d00) 2024-04-02T14:46:44.633 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: cache_index_and_filter_blocks: 1 2024-04-02T14:46:44.633 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-02T14:46:44.633 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-02T14:46:44.633 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: pin_top_level_index_and_filter: 1 2024-04-02T14:46:44.633 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: index_type: 0 2024-04-02T14:46:44.633 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: data_block_index_type: 0 2024-04-02T14:46:44.633 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: index_shortening: 1 2024-04-02T14:46:44.633 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-02T14:46:44.633 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: hash_index_allow_collision: 1 2024-04-02T14:46:44.633 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: checksum: 1 2024-04-02T14:46:44.633 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: no_block_cache: 0 2024-04-02T14:46:44.633 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: block_cache: 0x55c8fcfa4170 2024-04-02T14:46:44.633 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: block_cache_name: BinnedLRUCache 2024-04-02T14:46:44.633 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: block_cache_options: 2024-04-02T14:46:44.633 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: capacity : 536870912 2024-04-02T14:46:44.634 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: num_shard_bits : 4 2024-04-02T14:46:44.634 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: strict_capacity_limit : 0 2024-04-02T14:46:44.634 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: high_pri_pool_ratio: 0.000 2024-04-02T14:46:44.634 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: block_cache_compressed: (nil) 2024-04-02T14:46:44.634 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: persistent_cache: (nil) 2024-04-02T14:46:44.634 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: block_size: 4096 2024-04-02T14:46:44.634 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: block_size_deviation: 10 2024-04-02T14:46:44.634 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: block_restart_interval: 16 2024-04-02T14:46:44.634 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: index_block_restart_interval: 1 2024-04-02T14:46:44.634 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: metadata_block_size: 4096 2024-04-02T14:46:44.634 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: partition_filters: 0 2024-04-02T14:46:44.634 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: use_delta_encoding: 1 2024-04-02T14:46:44.634 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-02T14:46:44.634 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: whole_key_filtering: 1 2024-04-02T14:46:44.634 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: verify_compression: 0 2024-04-02T14:46:44.635 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: read_amp_bytes_per_bit: 0 2024-04-02T14:46:44.635 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: format_version: 4 2024-04-02T14:46:44.635 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: enable_index_compression: 1 2024-04-02T14:46:44.635 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: block_align: 0 2024-04-02T14:46:44.635 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.write_buffer_size: 33554432 2024-04-02T14:46:44.635 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_write_buffer_number: 2 2024-04-02T14:46:44.635 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compression: NoCompression 2024-04-02T14:46:44.635 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.bottommost_compression: Disabled 2024-04-02T14:46:44.635 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.prefix_extractor: nullptr 2024-04-02T14:46:44.635 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-02T14:46:44.635 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.num_levels: 7 2024-04-02T14:46:44.635 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-02T14:46:44.635 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-02T14:46:44.635 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-02T14:46:44.635 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-02T14:46:44.635 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-02T14:46:44.636 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-02T14:46:44.636 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-02T14:46:44.636 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-02T14:46:44.636 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-02T14:46:44.636 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-02T14:46:44.636 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-02T14:46:44.636 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compression_opts.level: 32767 2024-04-02T14:46:44.636 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compression_opts.strategy: 0 2024-04-02T14:46:44.636 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-02T14:46:44.636 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-02T14:46:44.636 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-02T14:46:44.636 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compression_opts.enabled: false 2024-04-02T14:46:44.636 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-02T14:46:44.636 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-02T14:46:44.636 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-02T14:46:44.636 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.target_file_size_base: 67108864 2024-04-02T14:46:44.636 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-02T14:46:44.637 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-02T14:46:44.637 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-02T14:46:44.637 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-02T14:46:44.637 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-02T14:46:44.637 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-02T14:46:44.637 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-02T14:46:44.637 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-02T14:46:44.637 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-02T14:46:44.637 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-02T14:46:44.637 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-02T14:46:44.637 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-02T14:46:44.637 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-02T14:46:44.637 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.arena_block_size: 4194304 2024-04-02T14:46:44.637 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-02T14:46:44.637 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-02T14:46:44.637 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-02T14:46:44.638 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.disable_auto_compactions: 0 2024-04-02T14:46:44.638 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-02T14:46:44.638 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-02T14:46:44.638 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-02T14:46:44.638 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-02T14:46:44.638 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-02T14:46:44.638 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-02T14:46:44.638 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-02T14:46:44.638 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-02T14:46:44.638 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-02T14:46:44.638 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-02T14:46:44.638 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.table_properties_collectors: 2024-04-02T14:46:44.638 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.inplace_update_support: 0 2024-04-02T14:46:44.638 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-02T14:46:44.638 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-02T14:46:44.638 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-02T14:46:44.638 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-02T14:46:44.639 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.bloom_locality: 0 2024-04-02T14:46:44.639 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.max_successive_merges: 0 2024-04-02T14:46:44.639 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-02T14:46:44.639 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.paranoid_file_checks: 0 2024-04-02T14:46:44.639 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.force_consistency_checks: 1 2024-04-02T14:46:44.639 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.report_bg_io_stats: 0 2024-04-02T14:46:44.639 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.ttl: 2592000 2024-04-02T14:46:44.639 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-02T14:46:44.639 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.enable_blob_files: false 2024-04-02T14:46:44.639 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.min_blob_size: 0 2024-04-02T14:46:44.639 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.blob_file_size: 268435456 2024-04-02T14:46:44.639 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-02T14:46:44.639 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-02T14:46:44.639 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-02T14:46:44.639 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi122/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-02T14:46:44.639 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-04-02T14:46:44.640 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-04-02T14:46:44.640 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712069204481339, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-02T14:46:44.640 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-04-02T14:46:44.640 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-02T14:46:44.640 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712069204482029, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79421, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77685, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71994, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712069204, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "95948831-5347-4b4e-87e1-ea84d111da00", "db_session_id": "V4BPC665WK85PWDFIKQS"}} 2024-04-02T14:46:44.640 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-04-02T14:46:44.640 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712069204482367, "job": 1, "event": "recovery_finished"} 2024-04-02T14:46:44.640 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi122/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-02T14:46:44.640 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55c8fcf8aa80 2024-04-02T14:46:44.640 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: DB pointer 0x55c8fcf9a000 2024-04-02T14:46:44.640 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-02T14:46:44.640 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-02T14:46:44.640 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: ** DB Stats ** 2024-04-02T14:46:44.640 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T14:46:44.640 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-02T14:46:44.640 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-02T14:46:44.641 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T14:46:44.641 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-02T14:46:44.641 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-02T14:46:44.641 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T14:46:44.641 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: 2024-04-02T14:46:44.641 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: ** Compaction Stats [default] ** 2024-04-02T14:46:44.641 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T14:46:44.641 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T14:46:44.641 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: L0 2/0 79.36 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 162.2 0.00 0.00 1 0.000 0 0 2024-04-02T14:46:44.641 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Sum 2/0 79.36 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 162.2 0.00 0.00 1 0.000 0 0 2024-04-02T14:46:44.641 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 162.2 0.00 0.00 1 0.000 0 0 2024-04-02T14:46:44.641 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: 2024-04-02T14:46:44.641 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: ** Compaction Stats [default] ** 2024-04-02T14:46:44.641 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T14:46:44.641 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T14:46:44.641 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 162.2 0.00 0.00 1 0.000 0 0 2024-04-02T14:46:44.641 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T14:46:44.642 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-02T14:46:44.642 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T14:46:44.642 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T14:46:44.642 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T14:46:44.642 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T14:46:44.642 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Cumulative compaction: 0.00 GB write, 26.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T14:46:44.642 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Interval compaction: 0.00 GB write, 26.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T14:46:44.642 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T14:46:44.642 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: 2024-04-02T14:46:44.642 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T14:46:44.642 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: 2024-04-02T14:46:44.642 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: ** Compaction Stats [default] ** 2024-04-02T14:46:44.642 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T14:46:44.642 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T14:46:44.643 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: L0 2/0 79.36 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 162.2 0.00 0.00 1 0.000 0 0 2024-04-02T14:46:44.643 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Sum 2/0 79.36 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 162.2 0.00 0.00 1 0.000 0 0 2024-04-02T14:46:44.643 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-02T14:46:44.643 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: 2024-04-02T14:46:44.643 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: ** Compaction Stats [default] ** 2024-04-02T14:46:44.643 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T14:46:44.643 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T14:46:44.643 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 162.2 0.00 0.00 1 0.000 0 0 2024-04-02T14:46:44.643 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T14:46:44.643 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-02T14:46:44.643 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T14:46:44.643 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T14:46:44.643 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T14:46:44.643 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T14:46:44.643 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Cumulative compaction: 0.00 GB write, 26.29 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T14:46:44.643 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T14:46:44.644 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T14:46:44.644 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: 2024-04-02T14:46:44.644 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T14:46:44.644 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: starting mon.smithi122 rank 0 at public addrs [v2:172.21.15.122:3300/0,v1:172.21.15.122:6789/0] at bind addrs [v2:172.21.15.122:3300/0,v1:172.21.15.122:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi122 fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 2024-04-02T14:46:44.644 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: mon.smithi122@-1(???) e1 preinit fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 2024-04-02T14:46:44.644 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: mon.smithi122@-1(???).mds e1 new map 2024-04-02T14:46:44.644 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: mon.smithi122@-1(???).mds e1 print_map 2024-04-02T14:46:44.644 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: e1 2024-04-02T14:46:44.644 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-02T14:46:44.644 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T14:46:44.644 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: legacy client fscid: -1 2024-04-02T14:46:44.644 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: 2024-04-02T14:46:44.644 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout: No filesystems configured 2024-04-02T14:46:44.644 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: mon.smithi122@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-02T14:46:44.644 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: mon.smithi122@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T14:46:44.644 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: mon.smithi122@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T14:46:44.645 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: mon.smithi122@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T14:46:44.645 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: mon.smithi122@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-02T14:46:44.645 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 bash[24783]: d8f91f766e220cb85c34a154aa046e9156ff779b665feb00ddfedd960176f3e9 2024-04-02T14:46:44.645 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 systemd[1]: Started Ceph mon.smithi122 for bc643104-f0ff-11ee-b647-cb9ed24678a4. 2024-04-02T14:46:44.645 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: mon.smithi122 is new leader, mons smithi122 in quorum (ranks 0) 2024-04-02T14:46:44.645 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: monmap e1: 1 mons at {smithi122=[v2:172.21.15.122:3300/0,v1:172.21.15.122:6789/0]} 2024-04-02T14:46:44.645 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: fsmap 2024-04-02T14:46:44.645 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: osdmap e1: 0 total, 0 up, 0 in 2024-04-02T14:46:44.645 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:44 smithi122 ceph-mon[24801]: mgrmap e1: no daemons active 2024-04-02T14:46:45.599 INFO:teuthology.orchestra.run.smithi122.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-02T14:46:45.599 INFO:teuthology.orchestra.run.smithi122.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-02T14:46:45.599 INFO:teuthology.orchestra.run.smithi122.stderr:Creating mgr... 2024-04-02T14:46:45.599 INFO:teuthology.orchestra.run.smithi122.stderr:Verifying port 9283 ... 2024-04-02T14:46:45.771 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: Failed to reset failed state of unit ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@mgr.smithi122.szjkfg.service: Unit ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@mgr.smithi122.szjkfg.service not loaded. 2024-04-02T14:46:45.778 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: Created symlink /etc/systemd/system/ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4.target.wants/ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@mgr.smithi122.szjkfg.service → /etc/systemd/system/ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@.service. 2024-04-02T14:46:46.244 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:46 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/1066871252' entity='client.admin' 2024-04-02T14:46:46.453 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: disabled 2024-04-02T14:46:46.460 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: inactive 2024-04-02T14:46:46.461 INFO:teuthology.orchestra.run.smithi122.stderr:firewalld.service is not enabled 2024-04-02T14:46:46.461 INFO:teuthology.orchestra.run.smithi122.stderr:Not possible to enable service . firewalld.service is not available 2024-04-02T14:46:46.469 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: disabled 2024-04-02T14:46:46.476 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: inactive 2024-04-02T14:46:46.477 INFO:teuthology.orchestra.run.smithi122.stderr:firewalld.service is not enabled 2024-04-02T14:46:46.477 INFO:teuthology.orchestra.run.smithi122.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-02T14:46:46.477 INFO:teuthology.orchestra.run.smithi122.stderr:Waiting for mgr to start... 2024-04-02T14:46:46.477 INFO:teuthology.orchestra.run.smithi122.stderr:Waiting for mgr... 2024-04-02T14:46:47.001 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: 2024-04-02T14:46:47.002 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: { 2024-04-02T14:46:47.002 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "fsid": "bc643104-f0ff-11ee-b647-cb9ed24678a4", 2024-04-02T14:46:47.002 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "health": { 2024-04-02T14:46:47.002 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-02T14:46:47.002 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "checks": {}, 2024-04-02T14:46:47.002 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "mutes": [] 2024-04-02T14:46:47.002 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: }, 2024-04-02T14:46:47.002 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-02T14:46:47.002 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "quorum": [ 2024-04-02T14:46:47.002 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: 0 2024-04-02T14:46:47.002 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: ], 2024-04-02T14:46:47.002 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-02T14:46:47.003 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "smithi122" 2024-04-02T14:46:47.003 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: ], 2024-04-02T14:46:47.003 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-04-02T14:46:47.003 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "monmap": { 2024-04-02T14:46:47.003 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T14:46:47.003 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-02T14:46:47.003 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-02T14:46:47.003 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: }, 2024-04-02T14:46:47.003 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "osdmap": { 2024-04-02T14:46:47.003 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T14:46:47.003 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-02T14:46:47.003 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-02T14:46:47.003 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-02T14:46:47.003 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-02T14:46:47.003 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-02T14:46:47.003 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-02T14:46:47.003 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: }, 2024-04-02T14:46:47.004 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "pgmap": { 2024-04-02T14:46:47.004 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-02T14:46:47.004 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-02T14:46:47.004 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-02T14:46:47.004 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-02T14:46:47.004 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-02T14:46:47.004 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-02T14:46:47.004 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-02T14:46:47.004 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-02T14:46:47.004 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: }, 2024-04-02T14:46:47.004 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "fsmap": { 2024-04-02T14:46:47.004 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T14:46:47.004 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-02T14:46:47.004 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-02T14:46:47.005 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: }, 2024-04-02T14:46:47.005 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-02T14:46:47.005 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "available": false, 2024-04-02T14:46:47.005 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-02T14:46:47.005 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "modules": [ 2024-04-02T14:46:47.005 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "iostat", 2024-04-02T14:46:47.005 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "nfs", 2024-04-02T14:46:47.005 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "restful" 2024-04-02T14:46:47.005 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: ], 2024-04-02T14:46:47.005 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "services": {} 2024-04-02T14:46:47.005 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: }, 2024-04-02T14:46:47.005 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "servicemap": { 2024-04-02T14:46:47.005 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T14:46:47.005 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "modified": "2024-04-02T14:46:40.135721+0000", 2024-04-02T14:46:47.005 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "services": {} 2024-04-02T14:46:47.005 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: }, 2024-04-02T14:46:47.006 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-02T14:46:47.006 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: } 2024-04-02T14:46:47.272 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:47 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/975758856' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-02T14:46:49.066 INFO:teuthology.orchestra.run.smithi122.stderr:mgr not available, waiting (1/15)... 2024-04-02T14:46:52.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:52 smithi122 ceph-mon[24801]: Activating manager daemon smithi122.szjkfg 2024-04-02T14:46:52.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:52 smithi122 ceph-mon[24801]: mgrmap e2: smithi122.szjkfg(active, starting, since 0.00273187s) 2024-04-02T14:46:52.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:52 smithi122 ceph-mon[24801]: from='mgr.14100 172.21.15.122:0/4071736030' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T14:46:52.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:52 smithi122 ceph-mon[24801]: from='mgr.14100 172.21.15.122:0/4071736030' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T14:46:52.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:52 smithi122 ceph-mon[24801]: from='mgr.14100 172.21.15.122:0/4071736030' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T14:46:52.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:52 smithi122 ceph-mon[24801]: from='mgr.14100 172.21.15.122:0/4071736030' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi122"}]: dispatch 2024-04-02T14:46:52.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:52 smithi122 ceph-mon[24801]: from='mgr.14100 172.21.15.122:0/4071736030' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mgr metadata", "who": "smithi122.szjkfg", "id": "smithi122.szjkfg"}]: dispatch 2024-04-02T14:46:52.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:52 smithi122 ceph-mon[24801]: Manager daemon smithi122.szjkfg is now available 2024-04-02T14:46:52.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:52 smithi122 ceph-mon[24801]: from='mgr.14100 172.21.15.122:0/4071736030' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:46:52.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:52 smithi122 ceph-mon[24801]: from='mgr.14100 172.21.15.122:0/4071736030' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/trash_purge_schedule"}]: dispatch 2024-04-02T14:46:52.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:52 smithi122 ceph-mon[24801]: from='mgr.14100 172.21.15.122:0/4071736030' entity='mgr.smithi122.szjkfg' 2024-04-02T14:46:52.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:52 smithi122 ceph-mon[24801]: from='mgr.14100 172.21.15.122:0/4071736030' entity='mgr.smithi122.szjkfg' 2024-04-02T14:46:52.593 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:52 smithi122 ceph-mon[24801]: from='mgr.14100 172.21.15.122:0/4071736030' entity='mgr.smithi122.szjkfg' 2024-04-02T14:46:54.586 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:54 smithi122 ceph-mon[24801]: mgrmap e3: smithi122.szjkfg(active, since 1.00695s) 2024-04-02T14:46:55.591 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:55 smithi122 ceph-mon[24801]: mgrmap e4: smithi122.szjkfg(active, since 2s) 2024-04-02T14:46:55.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:46:55 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/2326310224' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-02T14:46:55.670 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: 2024-04-02T14:46:55.670 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: { 2024-04-02T14:46:55.670 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "fsid": "bc643104-f0ff-11ee-b647-cb9ed24678a4", 2024-04-02T14:46:55.670 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "health": { 2024-04-02T14:46:55.670 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-02T14:46:55.670 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "checks": {}, 2024-04-02T14:46:55.670 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "mutes": [] 2024-04-02T14:46:55.670 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: }, 2024-04-02T14:46:55.670 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-02T14:46:55.670 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "quorum": [ 2024-04-02T14:46:55.670 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: 0 2024-04-02T14:46:55.670 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: ], 2024-04-02T14:46:55.670 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-02T14:46:55.671 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "smithi122" 2024-04-02T14:46:55.671 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: ], 2024-04-02T14:46:55.671 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "quorum_age": 10, 2024-04-02T14:46:55.671 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "monmap": { 2024-04-02T14:46:55.671 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T14:46:55.671 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-02T14:46:55.671 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-02T14:46:55.671 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: }, 2024-04-02T14:46:55.671 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "osdmap": { 2024-04-02T14:46:55.671 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T14:46:55.671 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-02T14:46:55.671 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-02T14:46:55.671 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-02T14:46:55.671 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-02T14:46:55.671 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-02T14:46:55.671 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-02T14:46:55.672 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: }, 2024-04-02T14:46:55.672 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "pgmap": { 2024-04-02T14:46:55.672 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-02T14:46:55.672 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-02T14:46:55.672 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-02T14:46:55.672 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-02T14:46:55.672 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-02T14:46:55.672 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-02T14:46:55.672 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-02T14:46:55.672 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-02T14:46:55.672 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: }, 2024-04-02T14:46:55.672 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "fsmap": { 2024-04-02T14:46:55.672 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T14:46:55.672 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-02T14:46:55.672 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-02T14:46:55.672 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: }, 2024-04-02T14:46:55.673 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-02T14:46:55.673 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "available": true, 2024-04-02T14:46:55.673 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-02T14:46:55.673 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "modules": [ 2024-04-02T14:46:55.673 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "iostat", 2024-04-02T14:46:55.673 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "nfs", 2024-04-02T14:46:55.673 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "restful" 2024-04-02T14:46:55.673 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: ], 2024-04-02T14:46:55.673 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "services": {} 2024-04-02T14:46:55.673 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: }, 2024-04-02T14:46:55.673 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "servicemap": { 2024-04-02T14:46:55.673 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T14:46:55.673 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "modified": "2024-04-02T14:46:40.135721+0000", 2024-04-02T14:46:55.673 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "services": {} 2024-04-02T14:46:55.673 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: }, 2024-04-02T14:46:55.673 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-02T14:46:55.673 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: } 2024-04-02T14:46:59.578 INFO:teuthology.orchestra.run.smithi122.stderr:mgr is available 2024-04-02T14:47:00.145 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: 2024-04-02T14:47:00.145 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: [global] 2024-04-02T14:47:00.145 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: fsid = bc643104-f0ff-11ee-b647-cb9ed24678a4 2024-04-02T14:47:00.145 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-02T14:47:00.145 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-02T14:47:00.145 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-02T14:47:00.145 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-02T14:47:00.145 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: 2024-04-02T14:47:00.145 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: [mgr] 2024-04-02T14:47:00.145 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-02T14:47:00.145 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: 2024-04-02T14:47:00.146 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: [osd] 2024-04-02T14:47:00.146 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-02T14:47:00.146 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-02T14:47:00.717 INFO:teuthology.orchestra.run.smithi122.stderr:Enabling cephadm module... 2024-04-02T14:47:00.824 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:00 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/2438895915' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-02T14:47:01.804 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:01 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/806209267' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-02T14:47:02.669 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:02 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/806209267' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-02T14:47:02.669 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:02 smithi122 ceph-mon[24801]: mgrmap e5: smithi122.szjkfg(active, since 9s) 2024-04-02T14:47:02.722 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: { 2024-04-02T14:47:02.722 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "epoch": 5, 2024-04-02T14:47:02.722 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "available": true, 2024-04-02T14:47:02.722 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "active_name": "smithi122.szjkfg", 2024-04-02T14:47:02.722 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-02T14:47:02.722 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: } 2024-04-02T14:47:03.271 INFO:teuthology.orchestra.run.smithi122.stderr:Waiting for the mgr to restart... 2024-04-02T14:47:03.271 INFO:teuthology.orchestra.run.smithi122.stderr:Waiting for mgr epoch 5... 2024-04-02T14:47:03.514 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:03 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/255732173' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-02T14:47:08.291 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: { 2024-04-02T14:47:08.291 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-04-02T14:47:08.291 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "initialized": true 2024-04-02T14:47:08.291 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: } 2024-04-02T14:47:08.571 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:08 smithi122 ceph-mon[24801]: Active manager daemon smithi122.szjkfg restarted 2024-04-02T14:47:08.572 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:08 smithi122 ceph-mon[24801]: Activating manager daemon smithi122.szjkfg 2024-04-02T14:47:08.572 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:08 smithi122 ceph-mon[24801]: osdmap e2: 0 total, 0 up, 0 in 2024-04-02T14:47:08.572 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:08 smithi122 ceph-mon[24801]: mgrmap e6: smithi122.szjkfg(active, starting, since 0.0525771s) 2024-04-02T14:47:08.572 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:08 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi122"}]: dispatch 2024-04-02T14:47:08.572 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:08 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mgr metadata", "who": "smithi122.szjkfg", "id": "smithi122.szjkfg"}]: dispatch 2024-04-02T14:47:08.572 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:08 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T14:47:08.572 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:08 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T14:47:08.572 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:08 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T14:47:08.572 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:08 smithi122 ceph-mon[24801]: Manager daemon smithi122.szjkfg is now available 2024-04-02T14:47:08.572 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:08 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:08.572 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:08 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:08.572 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:08 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:47:08.572 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:08 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:47:08.572 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:08 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:47:08.573 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:08 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:47:08.573 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:08 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/trash_purge_schedule"}]: dispatch 2024-04-02T14:47:08.573 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:08 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:08.573 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:08 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:47:08.808 INFO:teuthology.orchestra.run.smithi122.stderr:mgr epoch 5 is available 2024-04-02T14:47:08.808 INFO:teuthology.orchestra.run.smithi122.stderr:Setting orchestrator backend to cephadm... 2024-04-02T14:47:09.591 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:09 smithi122 ceph-mon[24801]: [02/Apr/2024:14:47:07] ENGINE Bus STARTING 2024-04-02T14:47:09.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:09 smithi122 ceph-mon[24801]: [02/Apr/2024:14:47:07] ENGINE Serving on https://172.21.15.122:7150 2024-04-02T14:47:09.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:09 smithi122 ceph-mon[24801]: [02/Apr/2024:14:47:07] ENGINE Bus STARTED 2024-04-02T14:47:09.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:09 smithi122 ceph-mon[24801]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-02T14:47:09.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:09 smithi122 ceph-mon[24801]: mgrmap e7: smithi122.szjkfg(active, since 1.05502s) 2024-04-02T14:47:09.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:09 smithi122 ceph-mon[24801]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-02T14:47:10.591 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:10 smithi122 ceph-mon[24801]: mgrmap e8: smithi122.szjkfg(active, since 2s) 2024-04-02T14:47:10.591 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:10 smithi122 ceph-mon[24801]: from='client.14128 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:47:10.591 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:10 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:10.592 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:10 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:47:10.599 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: value unchanged 2024-04-02T14:47:11.145 INFO:teuthology.orchestra.run.smithi122.stderr:Generating ssh key... 2024-04-02T14:47:11.841 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:11 smithi122 ceph-mon[24801]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:47:12.986 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:12 smithi122 ceph-mon[24801]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:47:12.987 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:12 smithi122 ceph-mon[24801]: Generating ssh key... 2024-04-02T14:47:12.987 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:12 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:12.987 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:12 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:12.987 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDD/Cwl5OJpMBbfaEam2bX6BhlRIeXWlV9z0Aw+sK34oEqI9V1a95WL0Y6yHG7rCrADMqhKcDl6KnJ4DQi7wRRZmpWK5/RrIWplW/j+mC2QPk9gu+XWcxl/burkVkxOZpzpVGB0Q7Voy+h47TXCjsx97UY7MZP8k93P0Xi5HfuPYEy5V4Q74WpqFmt4nl2KgaTSljf3n00BRlzNkd6RrgX3eEoVnP00ugQMhGT7HkxvXK8XhvrN6KjPEG3Tr+5vG7cHdBkxRdUNitu/XZ4/BTe7M7z5vhOVBT81u9lXXCy5AmhujwVqwDjHJJmjnTPTU6Fw3WL7kQ+knRAK37ySimrr9K/k0awPzpd5O5rL+LNiAaNudDg1ILfhQJYMiYwDrFe+oJOC41xmPnnTARGXIKGBrBO7vGoXvwdYtRW8yk/3ACyb1Xm5QoFgGKSRHqs17D1k9SQAOMIjldiHoNmhnWkFhZgC9El/uqU9OydUXDgbqhvq0isdr/3MHdl7irMMJkc= ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4 2024-04-02T14:47:13.850 INFO:teuthology.orchestra.run.smithi122.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-02T14:47:13.851 INFO:teuthology.orchestra.run.smithi122.stderr:Adding key to root@localhost authorized_keys... 2024-04-02T14:47:13.851 INFO:teuthology.orchestra.run.smithi122.stderr:Adding host smithi122... 2024-04-02T14:47:14.134 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:13 smithi122 ceph-mon[24801]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:47:15.397 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: Added host 'smithi122' with addr '172.21.15.122' 2024-04-02T14:47:15.737 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:15 smithi122 ceph-mon[24801]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi122", "addr": "172.21.15.122", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:47:15.737 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:15 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:15.738 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:15 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:47:15.998 INFO:teuthology.orchestra.run.smithi122.stderr:Deploying mon service with default placement... 2024-04-02T14:47:16.573 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:16 smithi122 ceph-mon[24801]: Deploying cephadm binary to smithi122 2024-04-02T14:47:16.573 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:16 smithi122 ceph-mon[24801]: Added host smithi122 2024-04-02T14:47:16.573 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-02T14:47:17.382 INFO:teuthology.orchestra.run.smithi122.stderr:Deploying mgr service with default placement... 2024-04-02T14:47:17.842 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:17 smithi122 ceph-mon[24801]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:47:17.842 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:17 smithi122 ceph-mon[24801]: Saving service mon spec with placement count:5 2024-04-02T14:47:17.843 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:17 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:18.018 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-02T14:47:19.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:19 smithi122 ceph-mon[24801]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:47:19.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:19 smithi122 ceph-mon[24801]: Saving service mgr spec with placement count:2 2024-04-02T14:47:19.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:19 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:19.342 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:19 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:19.387 INFO:teuthology.orchestra.run.smithi122.stderr:Deploying crash service with default placement... 2024-04-02T14:47:20.070 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: Scheduled crash update... 2024-04-02T14:47:20.815 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:20 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:20.815 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:20 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:21.213 INFO:teuthology.orchestra.run.smithi122.stderr:Deploying prometheus service with default placement... 2024-04-02T14:47:21.841 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:21 smithi122 ceph-mon[24801]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:47:21.841 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:21 smithi122 ceph-mon[24801]: Saving service crash spec with placement * 2024-04-02T14:47:21.841 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:21 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:21.907 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-04-02T14:47:22.838 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:22 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:22.838 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:22 smithi122 ceph-mon[24801]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:47:22.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:22 smithi122 ceph-mon[24801]: Saving service prometheus spec with placement count:1 2024-04-02T14:47:22.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:22 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:23.015 INFO:teuthology.orchestra.run.smithi122.stderr:Deploying grafana service with default placement... 2024-04-02T14:47:23.617 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-04-02T14:47:24.333 INFO:teuthology.orchestra.run.smithi122.stderr:Deploying node-exporter service with default placement... 2024-04-02T14:47:24.841 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:24 smithi122 ceph-mon[24801]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:47:24.841 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:24 smithi122 ceph-mon[24801]: Saving service grafana spec with placement count:1 2024-04-02T14:47:24.841 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:24 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:25.006 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-04-02T14:47:25.640 INFO:teuthology.orchestra.run.smithi122.stderr:Deploying alertmanager service with default placement... 2024-04-02T14:47:26.278 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:26 smithi122 ceph-mon[24801]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:47:26.279 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:26 smithi122 ceph-mon[24801]: Saving service node-exporter spec with placement * 2024-04-02T14:47:26.279 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:26 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:26.279 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-04-02T14:47:27.492 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:27 smithi122 ceph-mon[24801]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:47:27.492 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:27 smithi122 ceph-mon[24801]: Saving service alertmanager spec with placement count:1 2024-04-02T14:47:27.492 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:27 smithi122 ceph-mon[24801]: from='mgr.14116 172.21.15.122:0/543450444' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:28.503 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:28 smithi122 ceph-mon[24801]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:47:28.503 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:28 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/4028736868' entity='client.admin' 2024-04-02T14:47:29.300 INFO:teuthology.orchestra.run.smithi122.stderr:Enabling the dashboard module... 2024-04-02T14:47:29.901 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:29 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/2871203498' entity='client.admin' 2024-04-02T14:47:29.901 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:29 smithi122 ceph-mon[24801]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:47:31.022 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:30 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/191862406' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-02T14:47:31.841 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:31 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/191862406' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-02T14:47:31.841 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:31 smithi122 ceph-mon[24801]: mgrmap e9: smithi122.szjkfg(active, since 23s) 2024-04-02T14:47:32.013 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: { 2024-04-02T14:47:32.013 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "epoch": 9, 2024-04-02T14:47:32.013 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "available": true, 2024-04-02T14:47:32.013 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "active_name": "smithi122.szjkfg", 2024-04-02T14:47:32.013 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-02T14:47:32.013 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: } 2024-04-02T14:47:32.581 INFO:teuthology.orchestra.run.smithi122.stderr:Waiting for the mgr to restart... 2024-04-02T14:47:32.582 INFO:teuthology.orchestra.run.smithi122.stderr:Waiting for mgr epoch 9... 2024-04-02T14:47:32.761 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:32 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/615476791' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-02T14:47:37.000 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:36 smithi122 ceph-mon[24801]: Active manager daemon smithi122.szjkfg restarted 2024-04-02T14:47:37.000 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:36 smithi122 ceph-mon[24801]: Activating manager daemon smithi122.szjkfg 2024-04-02T14:47:37.000 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:36 smithi122 ceph-mon[24801]: osdmap e3: 0 total, 0 up, 0 in 2024-04-02T14:47:37.611 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: { 2024-04-02T14:47:37.611 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-04-02T14:47:37.611 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: "initialized": true 2024-04-02T14:47:37.611 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: } 2024-04-02T14:47:37.973 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:37 smithi122 ceph-mon[24801]: mgrmap e10: smithi122.szjkfg(active, starting, since 0.0552874s) 2024-04-02T14:47:37.973 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:37 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi122"}]: dispatch 2024-04-02T14:47:37.973 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:37 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mgr metadata", "who": "smithi122.szjkfg", "id": "smithi122.szjkfg"}]: dispatch 2024-04-02T14:47:37.973 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:37 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T14:47:37.973 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:37 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T14:47:37.973 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:37 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T14:47:37.973 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:37 smithi122 ceph-mon[24801]: Manager daemon smithi122.szjkfg is now available 2024-04-02T14:47:37.974 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:37 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:47:37.974 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:37 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:47:37.974 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:37 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:47:37.974 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:37 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/trash_purge_schedule"}]: dispatch 2024-04-02T14:47:37.974 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:37 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:38.258 INFO:teuthology.orchestra.run.smithi122.stderr:mgr epoch 9 is available 2024-04-02T14:47:38.259 INFO:teuthology.orchestra.run.smithi122.stderr:Generating a dashboard self-signed certificate... 2024-04-02T14:47:38.784 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:38 smithi122 ceph-mon[24801]: [02/Apr/2024:14:47:37] ENGINE Bus STARTING 2024-04-02T14:47:38.785 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:38 smithi122 ceph-mon[24801]: [02/Apr/2024:14:47:37] ENGINE Serving on https://172.21.15.122:7150 2024-04-02T14:47:38.785 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:38 smithi122 ceph-mon[24801]: [02/Apr/2024:14:47:37] ENGINE Bus STARTED 2024-04-02T14:47:38.785 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:38 smithi122 ceph-mon[24801]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-02T14:47:38.785 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:38 smithi122 ceph-mon[24801]: mgrmap e11: smithi122.szjkfg(active, since 1.06464s) 2024-04-02T14:47:38.785 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:38 smithi122 ceph-mon[24801]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-02T14:47:38.785 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:38 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:39.009 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-02T14:47:39.540 INFO:teuthology.orchestra.run.smithi122.stderr:Creating initial admin user... 2024-04-02T14:47:40.008 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:39 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:40.008 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:39 smithi122 ceph-mon[24801]: from='client.14170 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:47:40.008 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:39 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:40.008 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:39 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:40.009 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:39 smithi122 ceph-mon[24801]: mgrmap e12: smithi122.szjkfg(active, since 2s) 2024-04-02T14:47:40.448 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$bnwgf7jei3hj8Ss54ZHysu/mN4MY/hyyF5XMGyw/4ND1Sq1snawKy", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1712069260, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-02T14:47:41.645 INFO:teuthology.orchestra.run.smithi122.stderr:Fetching dashboard port number... 2024-04-02T14:47:41.767 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:41 smithi122 ceph-mon[24801]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:47:41.767 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:41 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:42.289 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: 8443 2024-04-02T14:47:42.781 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:42 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/4183127049' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-02T14:47:42.895 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: disabled 2024-04-02T14:47:42.902 INFO:teuthology.orchestra.run.smithi122.stderr:systemctl: inactive 2024-04-02T14:47:42.903 INFO:teuthology.orchestra.run.smithi122.stderr:firewalld.service is not enabled 2024-04-02T14:47:42.903 INFO:teuthology.orchestra.run.smithi122.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-02T14:47:42.904 INFO:teuthology.orchestra.run.smithi122.stderr:Ceph Dashboard is now available at: 2024-04-02T14:47:42.904 INFO:teuthology.orchestra.run.smithi122.stderr: 2024-04-02T14:47:42.905 INFO:teuthology.orchestra.run.smithi122.stderr: URL: https://smithi122.front.sepia.ceph.com:8443/ 2024-04-02T14:47:42.905 INFO:teuthology.orchestra.run.smithi122.stderr: User: admin 2024-04-02T14:47:42.905 INFO:teuthology.orchestra.run.smithi122.stderr: Password: z16rukqcr9 2024-04-02T14:47:42.905 INFO:teuthology.orchestra.run.smithi122.stderr: 2024-04-02T14:47:42.905 INFO:teuthology.orchestra.run.smithi122.stderr:Enabling autotune for osd_memory_target 2024-04-02T14:47:44.841 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:44 smithi122 ceph-mon[24801]: mgrmap e13: smithi122.szjkfg(active, since 6s) 2024-04-02T14:47:44.911 INFO:teuthology.orchestra.run.smithi122.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-04-02T14:47:45.525 INFO:teuthology.orchestra.run.smithi122.stderr:You can access the Ceph CLI with: 2024-04-02T14:47:45.525 INFO:teuthology.orchestra.run.smithi122.stderr: 2024-04-02T14:47:45.525 INFO:teuthology.orchestra.run.smithi122.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-02T14:47:45.525 INFO:teuthology.orchestra.run.smithi122.stderr: 2024-04-02T14:47:45.525 INFO:teuthology.orchestra.run.smithi122.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-02T14:47:45.525 INFO:teuthology.orchestra.run.smithi122.stderr: 2024-04-02T14:47:45.525 INFO:teuthology.orchestra.run.smithi122.stderr: ceph telemetry on 2024-04-02T14:47:45.525 INFO:teuthology.orchestra.run.smithi122.stderr: 2024-04-02T14:47:45.525 INFO:teuthology.orchestra.run.smithi122.stderr:For more information see: 2024-04-02T14:47:45.526 INFO:teuthology.orchestra.run.smithi122.stderr: 2024-04-02T14:47:45.526 INFO:teuthology.orchestra.run.smithi122.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-02T14:47:45.526 INFO:teuthology.orchestra.run.smithi122.stderr: 2024-04-02T14:47:45.526 INFO:teuthology.orchestra.run.smithi122.stderr:Bootstrap complete. 2024-04-02T14:47:45.561 INFO:tasks.cephadm:Fetching config... 2024-04-02T14:47:45.561 DEBUG:teuthology.orchestra.run.smithi122:> set -ex 2024-04-02T14:47:45.562 DEBUG:teuthology.orchestra.run.smithi122:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-02T14:47:45.577 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-02T14:47:45.577 DEBUG:teuthology.orchestra.run.smithi122:> set -ex 2024-04-02T14:47:45.578 DEBUG:teuthology.orchestra.run.smithi122:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-02T14:47:45.633 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-02T14:47:45.633 DEBUG:teuthology.orchestra.run.smithi122:> set -ex 2024-04-02T14:47:45.633 DEBUG:teuthology.orchestra.run.smithi122:> sudo dd if=/var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/keyring of=/dev/stdout 2024-04-02T14:47:45.699 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-02T14:47:45.699 DEBUG:teuthology.orchestra.run.smithi122:> set -ex 2024-04-02T14:47:45.699 DEBUG:teuthology.orchestra.run.smithi122:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-02T14:47:45.754 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-02T14:47:45.754 DEBUG:teuthology.orchestra.run.smithi122:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDD/Cwl5OJpMBbfaEam2bX6BhlRIeXWlV9z0Aw+sK34oEqI9V1a95WL0Y6yHG7rCrADMqhKcDl6KnJ4DQi7wRRZmpWK5/RrIWplW/j+mC2QPk9gu+XWcxl/burkVkxOZpzpVGB0Q7Voy+h47TXCjsx97UY7MZP8k93P0Xi5HfuPYEy5V4Q74WpqFmt4nl2KgaTSljf3n00BRlzNkd6RrgX3eEoVnP00ugQMhGT7HkxvXK8XhvrN6KjPEG3Tr+5vG7cHdBkxRdUNitu/XZ4/BTe7M7z5vhOVBT81u9lXXCy5AmhujwVqwDjHJJmjnTPTU6Fw3WL7kQ+knRAK37ySimrr9K/k0awPzpd5O5rL+LNiAaNudDg1ILfhQJYMiYwDrFe+oJOC41xmPnnTARGXIKGBrBO7vGoXvwdYtRW8yk/3ACyb1Xm5QoFgGKSRHqs17D1k9SQAOMIjldiHoNmhnWkFhZgC9El/uqU9OydUXDgbqhvq0isdr/3MHdl7irMMJkc= ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-02T14:47:45.836 INFO:teuthology.orchestra.run.smithi122.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDD/Cwl5OJpMBbfaEam2bX6BhlRIeXWlV9z0Aw+sK34oEqI9V1a95WL0Y6yHG7rCrADMqhKcDl6KnJ4DQi7wRRZmpWK5/RrIWplW/j+mC2QPk9gu+XWcxl/burkVkxOZpzpVGB0Q7Voy+h47TXCjsx97UY7MZP8k93P0Xi5HfuPYEy5V4Q74WpqFmt4nl2KgaTSljf3n00BRlzNkd6RrgX3eEoVnP00ugQMhGT7HkxvXK8XhvrN6KjPEG3Tr+5vG7cHdBkxRdUNitu/XZ4/BTe7M7z5vhOVBT81u9lXXCy5AmhujwVqwDjHJJmjnTPTU6Fw3WL7kQ+knRAK37ySimrr9K/k0awPzpd5O5rL+LNiAaNudDg1ILfhQJYMiYwDrFe+oJOC41xmPnnTARGXIKGBrBO7vGoXvwdYtRW8yk/3ACyb1Xm5QoFgGKSRHqs17D1k9SQAOMIjldiHoNmhnWkFhZgC9El/uqU9OydUXDgbqhvq0isdr/3MHdl7irMMJkc= ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4 2024-04-02T14:47:45.852 DEBUG:teuthology.orchestra.run.smithi183:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDD/Cwl5OJpMBbfaEam2bX6BhlRIeXWlV9z0Aw+sK34oEqI9V1a95WL0Y6yHG7rCrADMqhKcDl6KnJ4DQi7wRRZmpWK5/RrIWplW/j+mC2QPk9gu+XWcxl/burkVkxOZpzpVGB0Q7Voy+h47TXCjsx97UY7MZP8k93P0Xi5HfuPYEy5V4Q74WpqFmt4nl2KgaTSljf3n00BRlzNkd6RrgX3eEoVnP00ugQMhGT7HkxvXK8XhvrN6KjPEG3Tr+5vG7cHdBkxRdUNitu/XZ4/BTe7M7z5vhOVBT81u9lXXCy5AmhujwVqwDjHJJmjnTPTU6Fw3WL7kQ+knRAK37ySimrr9K/k0awPzpd5O5rL+LNiAaNudDg1ILfhQJYMiYwDrFe+oJOC41xmPnnTARGXIKGBrBO7vGoXvwdYtRW8yk/3ACyb1Xm5QoFgGKSRHqs17D1k9SQAOMIjldiHoNmhnWkFhZgC9El/uqU9OydUXDgbqhvq0isdr/3MHdl7irMMJkc= ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-02T14:47:45.900 INFO:teuthology.orchestra.run.smithi183.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDD/Cwl5OJpMBbfaEam2bX6BhlRIeXWlV9z0Aw+sK34oEqI9V1a95WL0Y6yHG7rCrADMqhKcDl6KnJ4DQi7wRRZmpWK5/RrIWplW/j+mC2QPk9gu+XWcxl/burkVkxOZpzpVGB0Q7Voy+h47TXCjsx97UY7MZP8k93P0Xi5HfuPYEy5V4Q74WpqFmt4nl2KgaTSljf3n00BRlzNkd6RrgX3eEoVnP00ugQMhGT7HkxvXK8XhvrN6KjPEG3Tr+5vG7cHdBkxRdUNitu/XZ4/BTe7M7z5vhOVBT81u9lXXCy5AmhujwVqwDjHJJmjnTPTU6Fw3WL7kQ+knRAK37ySimrr9K/k0awPzpd5O5rL+LNiAaNudDg1ILfhQJYMiYwDrFe+oJOC41xmPnnTARGXIKGBrBO7vGoXvwdYtRW8yk/3ACyb1Xm5QoFgGKSRHqs17D1k9SQAOMIjldiHoNmhnWkFhZgC9El/uqU9OydUXDgbqhvq0isdr/3MHdl7irMMJkc= ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4 2024-04-02T14:47:45.915 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-02T14:47:46.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:45 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/1000921003' entity='client.admin' 2024-04-02T14:47:49.623 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-02T14:47:49.623 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-02T14:47:50.151 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:49 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/1108999324' entity='client.admin' 2024-04-02T14:47:51.318 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:50 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:51.318 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:50 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd/host:smithi122", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:47:51.318 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:50 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:51.319 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:50 smithi122 ceph-mon[24801]: Deploying daemon alertmanager.smithi122 on smithi122 2024-04-02T14:47:52.213 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi183 2024-04-02T14:47:52.213 DEBUG:teuthology.orchestra.run.smithi183:> set -ex 2024-04-02T14:47:52.214 DEBUG:teuthology.orchestra.run.smithi183:> dd of=/etc/ceph/ceph.conf 2024-04-02T14:47:52.232 DEBUG:teuthology.orchestra.run.smithi183:> set -ex 2024-04-02T14:47:52.232 DEBUG:teuthology.orchestra.run.smithi183:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-02T14:47:52.289 INFO:tasks.cephadm:Adding host smithi183 to orchestrator... 2024-04-02T14:47:52.289 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph orch host add smithi183 2024-04-02T14:47:52.979 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:52 smithi122 ceph-mon[24801]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:47:52.979 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:52 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:56.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:55 smithi122 ceph-mon[24801]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi183", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:47:56.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:55 smithi122 ceph-mon[24801]: Deploying cephadm binary to smithi183 2024-04-02T14:47:56.175 INFO:teuthology.orchestra.run.smithi122.stdout:Added host 'smithi183' with addr '172.21.15.183' 2024-04-02T14:47:57.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:57 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:47:57.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:57 smithi122 ceph-mon[24801]: Added host smithi183 2024-04-02T14:47:57.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:57 smithi122 ceph-mon[24801]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:47:58.991 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph orch host ls --format=json 2024-04-02T14:47:59.899 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:47:59 smithi122 ceph-mon[24801]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:01.057 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:01 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:01.057 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:01 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi122", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-02T14:48:01.057 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:01 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi122", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-02T14:48:01.057 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:01 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:48:01.057 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:01 smithi122 ceph-mon[24801]: Deploying daemon crash.smithi122 on smithi122 2024-04-02T14:48:01.057 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:01 smithi122 ceph-mon[24801]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:01.346 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:48:01.346 INFO:teuthology.orchestra.run.smithi122.stdout:[{"addr": "172.21.15.122", "hostname": "smithi122", "labels": [], "status": ""}, {"addr": "172.21.15.183", "hostname": "smithi183", "labels": [], "status": ""}] 2024-04-02T14:48:02.925 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:02 smithi122 ceph-mon[24801]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T14:48:02.926 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:02 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:02.950 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-02T14:48:02.950 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph osd crush tunables default 2024-04-02T14:48:03.865 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:03 smithi122 ceph-mon[24801]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:04.921 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:04 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:04.921 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:04 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:04.921 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:04 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:04.921 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:04 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T14:48:04.922 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:04 smithi122 ceph-mon[24801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T14:48:04.922 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:04 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:04.922 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:04 smithi122 ceph-mon[24801]: Deploying daemon grafana.smithi122 on smithi122 2024-04-02T14:48:05.662 INFO:teuthology.orchestra.run.smithi122.stderr:adjusted tunables profile to default 2024-04-02T14:48:06.025 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:05 smithi122 ceph-mon[24801]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:06.025 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:05 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/2281629834' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-02T14:48:06.357 INFO:tasks.cephadm:Adding mon.smithi122 on smithi122 2024-04-02T14:48:06.357 INFO:tasks.cephadm:Adding mon.smithi183 on smithi183 2024-04-02T14:48:06.357 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph orch apply mon '2;smithi122:172.21.15.122=smithi122;smithi183:172.21.15.183=smithi183' 2024-04-02T14:48:07.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:06 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/2281629834' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-02T14:48:07.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:06 smithi122 ceph-mon[24801]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T14:48:07.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:06 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:08.041 INFO:teuthology.orchestra.run.smithi183.stdout:Scheduled mon update... 2024-04-02T14:48:08.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:07 smithi122 ceph-mon[24801]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:08.624 DEBUG:teuthology.orchestra.run.smithi183:mon.smithi183> sudo journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@mon.smithi183.service 2024-04-02T14:48:08.627 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T14:48:08.627 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T14:48:09.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:09 smithi122 ceph-mon[24801]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi122:172.21.15.122=smithi122;smithi183:172.21.15.183=smithi183", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:48:09.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:09 smithi122 ceph-mon[24801]: Saving service mon spec with placement smithi122:172.21.15.122=smithi122;smithi183:172.21.15.183=smithi183;count:2 2024-04-02T14:48:09.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:09 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:09.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:09 smithi122 ceph-mon[24801]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:10.385 INFO:teuthology.orchestra.run.smithi183.stdout: 2024-04-02T14:48:10.385 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":1,"fsid":"bc643104-f0ff-11ee-b647-cb9ed24678a4","modified":"2024-04-02T14:46:37.302523Z","created":"2024-04-02T14:46:37.302523Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi122","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:3300","nonce":0},{"type":"v1","addr":"172.21.15.122:6789","nonce":0}]},"addr":"172.21.15.122:6789/0","public_addr":"172.21.15.122:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T14:48:10.387 INFO:teuthology.orchestra.run.smithi183.stderr:dumped monmap epoch 1 2024-04-02T14:48:10.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:10 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/3283702308' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:48:11.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:11 smithi122 ceph-mon[24801]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:11.958 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T14:48:11.958 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T14:48:13.847 INFO:teuthology.orchestra.run.smithi183.stdout: 2024-04-02T14:48:13.847 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":1,"fsid":"bc643104-f0ff-11ee-b647-cb9ed24678a4","modified":"2024-04-02T14:46:37.302523Z","created":"2024-04-02T14:46:37.302523Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi122","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:3300","nonce":0},{"type":"v1","addr":"172.21.15.122:6789","nonce":0}]},"addr":"172.21.15.122:6789/0","public_addr":"172.21.15.122:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T14:48:13.849 INFO:teuthology.orchestra.run.smithi183.stderr:dumped monmap epoch 1 2024-04-02T14:48:14.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:13 smithi122 ceph-mon[24801]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:15.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:14 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/2649357792' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:48:15.378 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T14:48:15.379 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T14:48:16.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:15 smithi122 ceph-mon[24801]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:17.306 INFO:teuthology.orchestra.run.smithi183.stdout: 2024-04-02T14:48:17.307 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":1,"fsid":"bc643104-f0ff-11ee-b647-cb9ed24678a4","modified":"2024-04-02T14:46:37.302523Z","created":"2024-04-02T14:46:37.302523Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi122","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:3300","nonce":0},{"type":"v1","addr":"172.21.15.122:6789","nonce":0}]},"addr":"172.21.15.122:6789/0","public_addr":"172.21.15.122:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T14:48:17.309 INFO:teuthology.orchestra.run.smithi183.stderr:dumped monmap epoch 1 2024-04-02T14:48:18.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:17 smithi122 ceph-mon[24801]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:18.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:17 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/2899190791' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:48:18.918 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T14:48:18.919 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T14:48:19.819 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:19 smithi122 ceph-mon[24801]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:19.819 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:19 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:20.802 INFO:teuthology.orchestra.run.smithi183.stdout: 2024-04-02T14:48:20.802 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":1,"fsid":"bc643104-f0ff-11ee-b647-cb9ed24678a4","modified":"2024-04-02T14:46:37.302523Z","created":"2024-04-02T14:46:37.302523Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi122","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:3300","nonce":0},{"type":"v1","addr":"172.21.15.122:6789","nonce":0}]},"addr":"172.21.15.122:6789/0","public_addr":"172.21.15.122:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T14:48:20.804 INFO:teuthology.orchestra.run.smithi183.stderr:dumped monmap epoch 1 2024-04-02T14:48:20.981 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:20 smithi122 ceph-mon[24801]: Deploying daemon node-exporter.smithi122 on smithi122 2024-04-02T14:48:21.920 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:21 smithi122 ceph-mon[24801]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:21.920 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:21 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/2262382258' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:48:21.920 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:21 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:22.326 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T14:48:22.326 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T14:48:23.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:23 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:23.591 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:23 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-02T14:48:23.591 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:23 smithi122 ceph-mon[24801]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:24.294 INFO:teuthology.orchestra.run.smithi183.stdout: 2024-04-02T14:48:24.294 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":1,"fsid":"bc643104-f0ff-11ee-b647-cb9ed24678a4","modified":"2024-04-02T14:46:37.302523Z","created":"2024-04-02T14:46:37.302523Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi122","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:3300","nonce":0},{"type":"v1","addr":"172.21.15.122:6789","nonce":0}]},"addr":"172.21.15.122:6789/0","public_addr":"172.21.15.122:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T14:48:24.296 INFO:teuthology.orchestra.run.smithi183.stderr:dumped monmap epoch 1 2024-04-02T14:48:24.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:24 smithi122 ceph-mon[24801]: from='mgr.14158 172.21.15.122:0/1539357627' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-02T14:48:24.591 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:24 smithi122 ceph-mon[24801]: mgrmap e14: smithi122.szjkfg(active, since 46s) 2024-04-02T14:48:24.591 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:24 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/3991002628' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:48:25.890 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T14:48:25.890 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T14:48:27.754 INFO:teuthology.orchestra.run.smithi183.stdout: 2024-04-02T14:48:27.754 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":1,"fsid":"bc643104-f0ff-11ee-b647-cb9ed24678a4","modified":"2024-04-02T14:46:37.302523Z","created":"2024-04-02T14:46:37.302523Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi122","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:3300","nonce":0},{"type":"v1","addr":"172.21.15.122:6789","nonce":0}]},"addr":"172.21.15.122:6789/0","public_addr":"172.21.15.122:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T14:48:27.755 INFO:teuthology.orchestra.run.smithi183.stderr:dumped monmap epoch 1 2024-04-02T14:48:28.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:27 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/3695181152' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:48:29.350 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T14:48:29.350 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T14:48:30.381 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:30 smithi122 ceph-mon[24801]: Active manager daemon smithi122.szjkfg restarted 2024-04-02T14:48:30.381 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:30 smithi122 ceph-mon[24801]: Activating manager daemon smithi122.szjkfg 2024-04-02T14:48:30.381 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:30 smithi122 ceph-mon[24801]: osdmap e5: 0 total, 0 up, 0 in 2024-04-02T14:48:30.381 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:30 smithi122 ceph-mon[24801]: mgrmap e15: smithi122.szjkfg(active, starting, since 0.0522216s) 2024-04-02T14:48:30.381 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi122"}]: dispatch 2024-04-02T14:48:30.381 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mgr metadata", "who": "smithi122.szjkfg", "id": "smithi122.szjkfg"}]: dispatch 2024-04-02T14:48:30.381 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T14:48:30.381 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T14:48:30.381 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T14:48:30.381 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:30 smithi122 ceph-mon[24801]: Manager daemon smithi122.szjkfg is now available 2024-04-02T14:48:30.382 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:48:30.382 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:48:30.382 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:48:30.382 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:48:30.382 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:30.382 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:48:30.382 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/trash_purge_schedule"}]: dispatch 2024-04-02T14:48:30.382 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:31.203 INFO:teuthology.orchestra.run.smithi183.stdout: 2024-04-02T14:48:31.203 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":1,"fsid":"bc643104-f0ff-11ee-b647-cb9ed24678a4","modified":"2024-04-02T14:46:37.302523Z","created":"2024-04-02T14:46:37.302523Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi122","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:3300","nonce":0},{"type":"v1","addr":"172.21.15.122:6789","nonce":0}]},"addr":"172.21.15.122:6789/0","public_addr":"172.21.15.122:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T14:48:31.205 INFO:teuthology.orchestra.run.smithi183.stderr:dumped monmap epoch 1 2024-04-02T14:48:31.585 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:31 smithi122 ceph-mon[24801]: [02/Apr/2024:14:48:29] ENGINE Bus STARTING 2024-04-02T14:48:31.585 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:31 smithi122 ceph-mon[24801]: [02/Apr/2024:14:48:29] ENGINE Serving on https://172.21.15.122:7150 2024-04-02T14:48:31.585 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:31 smithi122 ceph-mon[24801]: [02/Apr/2024:14:48:29] ENGINE Bus STARTED 2024-04-02T14:48:31.585 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:31 smithi122 ceph-mon[24801]: mgrmap e16: smithi122.szjkfg(active, since 1.05472s) 2024-04-02T14:48:31.585 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:31 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:31.585 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:31 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:32.423 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:32 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/1373131495' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:48:32.423 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:32.423 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:32 smithi122 ceph-mon[24801]: mgrmap e17: smithi122.szjkfg(active, since 2s) 2024-04-02T14:48:32.784 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T14:48:32.784 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T14:48:33.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:33 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:33.841 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:33 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:33.841 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:33 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd/host:smithi122", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:48:33.841 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:33 smithi122 ceph-mon[24801]: Updating smithi122:/etc/ceph/ceph.conf 2024-04-02T14:48:35.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:34 smithi122 ceph-mon[24801]: Updating smithi122:/etc/ceph/ceph.client.admin.keyring 2024-04-02T14:48:35.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:34 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:35.562 INFO:teuthology.orchestra.run.smithi183.stdout: 2024-04-02T14:48:35.563 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":1,"fsid":"bc643104-f0ff-11ee-b647-cb9ed24678a4","modified":"2024-04-02T14:46:37.302523Z","created":"2024-04-02T14:46:37.302523Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi122","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:3300","nonce":0},{"type":"v1","addr":"172.21.15.122:6789","nonce":0}]},"addr":"172.21.15.122:6789/0","public_addr":"172.21.15.122:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T14:48:35.564 INFO:teuthology.orchestra.run.smithi183.stderr:dumped monmap epoch 1 2024-04-02T14:48:36.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:35 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:36.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:35 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:36.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:35 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd/host:smithi183", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:48:36.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:35 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:36.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:35 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi183", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-02T14:48:36.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:35 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi183", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-02T14:48:36.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:35 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:48:36.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:35 smithi122 ceph-mon[24801]: Deploying daemon crash.smithi183 on smithi183 2024-04-02T14:48:36.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:35 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/3700434897' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:48:37.138 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T14:48:37.138 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T14:48:39.176 INFO:teuthology.orchestra.run.smithi183.stdout: 2024-04-02T14:48:39.176 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":1,"fsid":"bc643104-f0ff-11ee-b647-cb9ed24678a4","modified":"2024-04-02T14:46:37.302523Z","created":"2024-04-02T14:46:37.302523Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi122","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:3300","nonce":0},{"type":"v1","addr":"172.21.15.122:6789","nonce":0}]},"addr":"172.21.15.122:6789/0","public_addr":"172.21.15.122:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T14:48:39.177 INFO:teuthology.orchestra.run.smithi183.stderr:dumped monmap epoch 1 2024-04-02T14:48:39.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:39 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/1157583997' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:48:40.782 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T14:48:40.783 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T14:48:41.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:41 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:41.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:41 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi183.smxedq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T14:48:41.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:41 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi183.smxedq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-02T14:48:41.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:41 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T14:48:41.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:41 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:48:41.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:41 smithi122 ceph-mon[24801]: Deploying daemon mgr.smithi183.smxedq on smithi183 2024-04-02T14:48:43.191 INFO:teuthology.orchestra.run.smithi183.stdout: 2024-04-02T14:48:43.191 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":1,"fsid":"bc643104-f0ff-11ee-b647-cb9ed24678a4","modified":"2024-04-02T14:46:37.302523Z","created":"2024-04-02T14:46:37.302523Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi122","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:3300","nonce":0},{"type":"v1","addr":"172.21.15.122:6789","nonce":0}]},"addr":"172.21.15.122:6789/0","public_addr":"172.21.15.122:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T14:48:43.192 INFO:teuthology.orchestra.run.smithi183.stderr:dumped monmap epoch 1 2024-04-02T14:48:43.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:43 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/3361845857' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:48:45.055 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T14:48:45.055 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T14:48:45.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:45.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T14:48:45.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:48:45.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:44 smithi122 ceph-mon[24801]: Deploying daemon mon.smithi183 on smithi183 2024-04-02T14:48:45.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:48.040 INFO:teuthology.orchestra.run.smithi183.stdout: 2024-04-02T14:48:48.040 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":1,"fsid":"bc643104-f0ff-11ee-b647-cb9ed24678a4","modified":"2024-04-02T14:46:37.302523Z","created":"2024-04-02T14:46:37.302523Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi122","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:3300","nonce":0},{"type":"v1","addr":"172.21.15.122:6789","nonce":0}]},"addr":"172.21.15.122:6789/0","public_addr":"172.21.15.122:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T14:48:48.041 INFO:teuthology.orchestra.run.smithi183.stderr:dumped monmap epoch 1 2024-04-02T14:48:48.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:48 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/4091934788' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:48:50.175 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T14:48:50.177 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T14:48:54.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi183"}]: dispatch 2024-04-02T14:48:54.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi122"}]: dispatch 2024-04-02T14:48:54.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: mon.smithi122 calling monitor election 2024-04-02T14:48:54.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi183"}]: dispatch 2024-04-02T14:48:54.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:54.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: from='mgr.? 172.21.15.183:0/2109326057' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi183.smxedq/crt"}]: dispatch 2024-04-02T14:48:54.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi183"}]: dispatch 2024-04-02T14:48:54.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi183"}]: dispatch 2024-04-02T14:48:54.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: mon.smithi183 calling monitor election 2024-04-02T14:48:54.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:54.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi183"}]: dispatch 2024-04-02T14:48:54.092 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi183"}]: dispatch 2024-04-02T14:48:54.092 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:54.092 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi183"}]: dispatch 2024-04-02T14:48:54.092 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: mon.smithi122 is new leader, mons smithi122,smithi183 in quorum (ranks 0,1) 2024-04-02T14:48:54.092 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: monmap e2: 2 mons at {smithi122=[v2:172.21.15.122:3300/0,v1:172.21.15.122:6789/0],smithi183=[v2:172.21.15.183:3300/0,v1:172.21.15.183:6789/0]} 2024-04-02T14:48:54.092 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: fsmap 2024-04-02T14:48:54.092 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: osdmap e5: 0 total, 0 up, 0 in 2024-04-02T14:48:54.092 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: mgrmap e17: smithi122.szjkfg(active, since 24s) 2024-04-02T14:48:54.092 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: Standby manager daemon smithi183.smxedq started 2024-04-02T14:48:54.092 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: from='mgr.? 172.21.15.183:0/2109326057' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T14:48:54.092 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: overall HEALTH_OK 2024-04-02T14:48:54.092 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: from='mgr.? 172.21.15.183:0/2109326057' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi183.smxedq/key"}]: dispatch 2024-04-02T14:48:54.092 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: from='mgr.? 172.21.15.183:0/2109326057' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T14:48:54.092 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:53 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:55.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:54 smithi122 ceph-mon[24801]: Deploying daemon node-exporter.smithi183 on smithi183 2024-04-02T14:48:55.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:54 smithi122 ceph-mon[24801]: mgrmap e18: smithi122.szjkfg(active, since 24s), standbys: smithi183.smxedq 2024-04-02T14:48:55.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:54 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mgr metadata", "who": "smithi183.smxedq", "id": "smithi183.smxedq"}]: dispatch 2024-04-02T14:48:55.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:54 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:55.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:54 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi183"}]: dispatch 2024-04-02T14:48:55.149 INFO:teuthology.orchestra.run.smithi183.stdout: 2024-04-02T14:48:55.149 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":2,"fsid":"bc643104-f0ff-11ee-b647-cb9ed24678a4","modified":"2024-04-02T14:48:48.571388Z","created":"2024-04-02T14:46:37.302523Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi122","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:3300","nonce":0},{"type":"v1","addr":"172.21.15.122:6789","nonce":0}]},"addr":"172.21.15.122:6789/0","public_addr":"172.21.15.122:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi183","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:3300","nonce":0},{"type":"v1","addr":"172.21.15.183:6789","nonce":0}]},"addr":"172.21.15.183:6789/0","public_addr":"172.21.15.183:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-02T14:48:55.150 INFO:teuthology.orchestra.run.smithi183.stderr:dumped monmap epoch 2 2024-04-02T14:48:55.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:48:55 smithi183 ceph-mon[27058]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:56.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:55 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/4215378425' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T14:48:56.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:55 smithi122 ceph-mon[24801]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:56.423 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-02T14:48:56.423 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph config generate-minimal-conf 2024-04-02T14:48:58.377 INFO:teuthology.orchestra.run.smithi122.stdout:# minimal ceph.conf for bc643104-f0ff-11ee-b647-cb9ed24678a4 2024-04-02T14:48:58.377 INFO:teuthology.orchestra.run.smithi122.stdout:[global] 2024-04-02T14:48:58.377 INFO:teuthology.orchestra.run.smithi122.stdout: fsid = bc643104-f0ff-11ee-b647-cb9ed24678a4 2024-04-02T14:48:58.377 INFO:teuthology.orchestra.run.smithi122.stdout: mon_host = [v2:172.21.15.122:3300/0,v1:172.21.15.122:6789/0] [v2:172.21.15.183:3300/0,v1:172.21.15.183:6789/0] 2024-04-02T14:48:58.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:58 smithi122 ceph-mon[24801]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:58.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:58 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:58.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:58 smithi122 ceph-mon[24801]: Deploying daemon prometheus.smithi122 on smithi122 2024-04-02T14:48:58.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:48:58 smithi183 ceph-mon[27058]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:48:58.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:48:58 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:58.652 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:48:58 smithi183 ceph-mon[27058]: Deploying daemon prometheus.smithi122 on smithi122 2024-04-02T14:48:58.946 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-02T14:48:58.946 DEBUG:teuthology.orchestra.run.smithi122:> set -ex 2024-04-02T14:48:58.946 DEBUG:teuthology.orchestra.run.smithi122:> sudo dd of=/etc/ceph/ceph.conf 2024-04-02T14:48:58.975 DEBUG:teuthology.orchestra.run.smithi122:> set -ex 2024-04-02T14:48:58.976 DEBUG:teuthology.orchestra.run.smithi122:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-02T14:48:59.044 DEBUG:teuthology.orchestra.run.smithi183:> set -ex 2024-04-02T14:48:59.044 DEBUG:teuthology.orchestra.run.smithi183:> sudo dd of=/etc/ceph/ceph.conf 2024-04-02T14:48:59.080 DEBUG:teuthology.orchestra.run.smithi183:> set -ex 2024-04-02T14:48:59.081 DEBUG:teuthology.orchestra.run.smithi183:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-02T14:48:59.150 INFO:tasks.cephadm:Deploying OSDs... 2024-04-02T14:48:59.150 DEBUG:teuthology.orchestra.run.smithi122:> set -ex 2024-04-02T14:48:59.150 DEBUG:teuthology.orchestra.run.smithi122:> dd if=/scratch_devs of=/dev/stdout 2024-04-02T14:48:59.165 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-02T14:48:59.165 DEBUG:teuthology.orchestra.run.smithi122:> stat /dev/vg_nvme/lv_1 2024-04-02T14:48:59.220 INFO:teuthology.orchestra.run.smithi122.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-02T14:48:59.220 INFO:teuthology.orchestra.run.smithi122.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T14:48:59.220 INFO:teuthology.orchestra.run.smithi122.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-02T14:48:59.220 INFO:teuthology.orchestra.run.smithi122.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T14:48:59.221 INFO:teuthology.orchestra.run.smithi122.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T14:48:59.221 INFO:teuthology.orchestra.run.smithi122.stdout:Access: 2024-04-02 14:48:19.940646474 +0000 2024-04-02T14:48:59.221 INFO:teuthology.orchestra.run.smithi122.stdout:Modify: 2024-04-02 14:45:38.386894554 +0000 2024-04-02T14:48:59.221 INFO:teuthology.orchestra.run.smithi122.stdout:Change: 2024-04-02 14:45:38.386894554 +0000 2024-04-02T14:48:59.221 INFO:teuthology.orchestra.run.smithi122.stdout: Birth: 2024-04-02 14:45:38.386894554 +0000 2024-04-02T14:48:59.221 DEBUG:teuthology.orchestra.run.smithi122:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-02T14:48:59.286 INFO:teuthology.orchestra.run.smithi122.stderr:1+0 records in 2024-04-02T14:48:59.286 INFO:teuthology.orchestra.run.smithi122.stderr:1+0 records out 2024-04-02T14:48:59.286 INFO:teuthology.orchestra.run.smithi122.stderr:512 bytes copied, 0.000194346 s, 2.6 MB/s 2024-04-02T14:48:59.287 DEBUG:teuthology.orchestra.run.smithi122:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-02T14:48:59.342 DEBUG:teuthology.orchestra.run.smithi122:> stat /dev/vg_nvme/lv_2 2024-04-02T14:48:59.396 INFO:teuthology.orchestra.run.smithi122.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-02T14:48:59.397 INFO:teuthology.orchestra.run.smithi122.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T14:48:59.397 INFO:teuthology.orchestra.run.smithi122.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-04-02T14:48:59.397 INFO:teuthology.orchestra.run.smithi122.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T14:48:59.397 INFO:teuthology.orchestra.run.smithi122.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T14:48:59.397 INFO:teuthology.orchestra.run.smithi122.stdout:Access: 2024-04-02 14:48:19.941646450 +0000 2024-04-02T14:48:59.397 INFO:teuthology.orchestra.run.smithi122.stdout:Modify: 2024-04-02 14:45:38.735885965 +0000 2024-04-02T14:48:59.397 INFO:teuthology.orchestra.run.smithi122.stdout:Change: 2024-04-02 14:45:38.735885965 +0000 2024-04-02T14:48:59.397 INFO:teuthology.orchestra.run.smithi122.stdout: Birth: 2024-04-02 14:45:38.735885965 +0000 2024-04-02T14:48:59.397 DEBUG:teuthology.orchestra.run.smithi122:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-02T14:48:59.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:48:59 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/220860248' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:48:59.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:48:59 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:59.462 INFO:teuthology.orchestra.run.smithi122.stderr:1+0 records in 2024-04-02T14:48:59.462 INFO:teuthology.orchestra.run.smithi122.stderr:1+0 records out 2024-04-02T14:48:59.463 INFO:teuthology.orchestra.run.smithi122.stderr:512 bytes copied, 0.000175306 s, 2.9 MB/s 2024-04-02T14:48:59.464 DEBUG:teuthology.orchestra.run.smithi122:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-02T14:48:59.519 DEBUG:teuthology.orchestra.run.smithi122:> stat /dev/vg_nvme/lv_3 2024-04-02T14:48:59.573 INFO:teuthology.orchestra.run.smithi122.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-02T14:48:59.574 INFO:teuthology.orchestra.run.smithi122.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T14:48:59.574 INFO:teuthology.orchestra.run.smithi122.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-04-02T14:48:59.574 INFO:teuthology.orchestra.run.smithi122.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T14:48:59.574 INFO:teuthology.orchestra.run.smithi122.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T14:48:59.574 INFO:teuthology.orchestra.run.smithi122.stdout:Access: 2024-04-02 14:48:19.941646450 +0000 2024-04-02T14:48:59.574 INFO:teuthology.orchestra.run.smithi122.stdout:Modify: 2024-04-02 14:45:39.073877648 +0000 2024-04-02T14:48:59.574 INFO:teuthology.orchestra.run.smithi122.stdout:Change: 2024-04-02 14:45:39.073877648 +0000 2024-04-02T14:48:59.574 INFO:teuthology.orchestra.run.smithi122.stdout: Birth: 2024-04-02 14:45:39.073877648 +0000 2024-04-02T14:48:59.574 DEBUG:teuthology.orchestra.run.smithi122:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-02T14:48:59.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:59 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/220860248' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:48:59.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:48:59 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:48:59.616 INFO:teuthology.orchestra.run.smithi122.stderr:1+0 records in 2024-04-02T14:48:59.616 INFO:teuthology.orchestra.run.smithi122.stderr:1+0 records out 2024-04-02T14:48:59.616 INFO:teuthology.orchestra.run.smithi122.stderr:512 bytes copied, 0.00019613 s, 2.6 MB/s 2024-04-02T14:48:59.617 DEBUG:teuthology.orchestra.run.smithi122:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-02T14:48:59.672 DEBUG:teuthology.orchestra.run.smithi122:> stat /dev/vg_nvme/lv_4 2024-04-02T14:48:59.727 INFO:teuthology.orchestra.run.smithi122.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-02T14:48:59.727 INFO:teuthology.orchestra.run.smithi122.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T14:48:59.728 INFO:teuthology.orchestra.run.smithi122.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-04-02T14:48:59.728 INFO:teuthology.orchestra.run.smithi122.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T14:48:59.728 INFO:teuthology.orchestra.run.smithi122.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T14:48:59.728 INFO:teuthology.orchestra.run.smithi122.stdout:Access: 2024-04-02 14:48:19.941646450 +0000 2024-04-02T14:48:59.728 INFO:teuthology.orchestra.run.smithi122.stdout:Modify: 2024-04-02 14:45:39.428868912 +0000 2024-04-02T14:48:59.728 INFO:teuthology.orchestra.run.smithi122.stdout:Change: 2024-04-02 14:45:39.428868912 +0000 2024-04-02T14:48:59.728 INFO:teuthology.orchestra.run.smithi122.stdout: Birth: 2024-04-02 14:45:39.428868912 +0000 2024-04-02T14:48:59.728 DEBUG:teuthology.orchestra.run.smithi122:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-02T14:48:59.793 INFO:teuthology.orchestra.run.smithi122.stderr:1+0 records in 2024-04-02T14:48:59.793 INFO:teuthology.orchestra.run.smithi122.stderr:1+0 records out 2024-04-02T14:48:59.793 INFO:teuthology.orchestra.run.smithi122.stderr:512 bytes copied, 0.000176921 s, 2.9 MB/s 2024-04-02T14:48:59.794 DEBUG:teuthology.orchestra.run.smithi122:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-02T14:48:59.850 DEBUG:teuthology.orchestra.run.smithi183:> set -ex 2024-04-02T14:48:59.850 DEBUG:teuthology.orchestra.run.smithi183:> dd if=/scratch_devs of=/dev/stdout 2024-04-02T14:48:59.869 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-02T14:48:59.869 DEBUG:teuthology.orchestra.run.smithi183:> stat /dev/vg_nvme/lv_1 2024-04-02T14:48:59.927 INFO:teuthology.orchestra.run.smithi183.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-02T14:48:59.927 INFO:teuthology.orchestra.run.smithi183.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T14:48:59.927 INFO:teuthology.orchestra.run.smithi183.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-04-02T14:48:59.927 INFO:teuthology.orchestra.run.smithi183.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T14:48:59.928 INFO:teuthology.orchestra.run.smithi183.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T14:48:59.928 INFO:teuthology.orchestra.run.smithi183.stdout:Access: 2024-04-02 14:48:54.168462226 +0000 2024-04-02T14:48:59.928 INFO:teuthology.orchestra.run.smithi183.stdout:Modify: 2024-04-02 14:45:43.671891287 +0000 2024-04-02T14:48:59.928 INFO:teuthology.orchestra.run.smithi183.stdout:Change: 2024-04-02 14:45:43.671891287 +0000 2024-04-02T14:48:59.928 INFO:teuthology.orchestra.run.smithi183.stdout: Birth: 2024-04-02 14:45:43.671891287 +0000 2024-04-02T14:48:59.928 DEBUG:teuthology.orchestra.run.smithi183:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-02T14:48:59.997 INFO:teuthology.orchestra.run.smithi183.stderr:1+0 records in 2024-04-02T14:48:59.998 INFO:teuthology.orchestra.run.smithi183.stderr:1+0 records out 2024-04-02T14:48:59.998 INFO:teuthology.orchestra.run.smithi183.stderr:512 bytes copied, 0.000215691 s, 2.4 MB/s 2024-04-02T14:48:59.999 DEBUG:teuthology.orchestra.run.smithi183:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-02T14:49:00.057 DEBUG:teuthology.orchestra.run.smithi183:> stat /dev/vg_nvme/lv_2 2024-04-02T14:49:00.113 INFO:teuthology.orchestra.run.smithi183.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-02T14:49:00.113 INFO:teuthology.orchestra.run.smithi183.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T14:49:00.114 INFO:teuthology.orchestra.run.smithi183.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-04-02T14:49:00.114 INFO:teuthology.orchestra.run.smithi183.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T14:49:00.114 INFO:teuthology.orchestra.run.smithi183.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T14:49:00.114 INFO:teuthology.orchestra.run.smithi183.stdout:Access: 2024-04-02 14:48:54.168462226 +0000 2024-04-02T14:49:00.114 INFO:teuthology.orchestra.run.smithi183.stdout:Modify: 2024-04-02 14:45:44.021881942 +0000 2024-04-02T14:49:00.114 INFO:teuthology.orchestra.run.smithi183.stdout:Change: 2024-04-02 14:45:44.021881942 +0000 2024-04-02T14:49:00.114 INFO:teuthology.orchestra.run.smithi183.stdout: Birth: 2024-04-02 14:45:44.021881942 +0000 2024-04-02T14:49:00.114 DEBUG:teuthology.orchestra.run.smithi183:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-02T14:49:00.182 INFO:teuthology.orchestra.run.smithi183.stderr:1+0 records in 2024-04-02T14:49:00.182 INFO:teuthology.orchestra.run.smithi183.stderr:1+0 records out 2024-04-02T14:49:00.182 INFO:teuthology.orchestra.run.smithi183.stderr:512 bytes copied, 0.000207079 s, 2.5 MB/s 2024-04-02T14:49:00.183 DEBUG:teuthology.orchestra.run.smithi183:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-02T14:49:00.241 DEBUG:teuthology.orchestra.run.smithi183:> stat /dev/vg_nvme/lv_3 2024-04-02T14:49:00.298 INFO:teuthology.orchestra.run.smithi183.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-02T14:49:00.298 INFO:teuthology.orchestra.run.smithi183.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T14:49:00.298 INFO:teuthology.orchestra.run.smithi183.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-04-02T14:49:00.298 INFO:teuthology.orchestra.run.smithi183.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T14:49:00.298 INFO:teuthology.orchestra.run.smithi183.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T14:49:00.298 INFO:teuthology.orchestra.run.smithi183.stdout:Access: 2024-04-02 14:48:54.169462199 +0000 2024-04-02T14:49:00.298 INFO:teuthology.orchestra.run.smithi183.stdout:Modify: 2024-04-02 14:45:44.363872812 +0000 2024-04-02T14:49:00.298 INFO:teuthology.orchestra.run.smithi183.stdout:Change: 2024-04-02 14:45:44.363872812 +0000 2024-04-02T14:49:00.299 INFO:teuthology.orchestra.run.smithi183.stdout: Birth: 2024-04-02 14:45:44.363872812 +0000 2024-04-02T14:49:00.299 DEBUG:teuthology.orchestra.run.smithi183:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-02T14:49:00.366 INFO:teuthology.orchestra.run.smithi183.stderr:1+0 records in 2024-04-02T14:49:00.366 INFO:teuthology.orchestra.run.smithi183.stderr:1+0 records out 2024-04-02T14:49:00.366 INFO:teuthology.orchestra.run.smithi183.stderr:512 bytes copied, 0.000213124 s, 2.4 MB/s 2024-04-02T14:49:00.367 DEBUG:teuthology.orchestra.run.smithi183:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-02T14:49:00.424 DEBUG:teuthology.orchestra.run.smithi183:> stat /dev/vg_nvme/lv_4 2024-04-02T14:49:00.480 INFO:teuthology.orchestra.run.smithi183.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-02T14:49:00.480 INFO:teuthology.orchestra.run.smithi183.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T14:49:00.480 INFO:teuthology.orchestra.run.smithi183.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-04-02T14:49:00.481 INFO:teuthology.orchestra.run.smithi183.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T14:49:00.481 INFO:teuthology.orchestra.run.smithi183.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T14:49:00.481 INFO:teuthology.orchestra.run.smithi183.stdout:Access: 2024-04-02 14:48:54.169462199 +0000 2024-04-02T14:49:00.481 INFO:teuthology.orchestra.run.smithi183.stdout:Modify: 2024-04-02 14:45:44.703863734 +0000 2024-04-02T14:49:00.481 INFO:teuthology.orchestra.run.smithi183.stdout:Change: 2024-04-02 14:45:44.703863734 +0000 2024-04-02T14:49:00.481 INFO:teuthology.orchestra.run.smithi183.stdout: Birth: 2024-04-02 14:45:44.703863734 +0000 2024-04-02T14:49:00.481 DEBUG:teuthology.orchestra.run.smithi183:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-02T14:49:00.545 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:00 smithi183 ceph-mon[27058]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:00.547 INFO:teuthology.orchestra.run.smithi183.stderr:1+0 records in 2024-04-02T14:49:00.547 INFO:teuthology.orchestra.run.smithi183.stderr:1+0 records out 2024-04-02T14:49:00.547 INFO:teuthology.orchestra.run.smithi183.stderr:512 bytes copied, 0.000213355 s, 2.4 MB/s 2024-04-02T14:49:00.548 DEBUG:teuthology.orchestra.run.smithi183:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-02T14:49:00.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:00 smithi122 ceph-mon[24801]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:00.605 INFO:tasks.cephadm:Deploying osd.0 on smithi122 with /dev/vg_nvme/lv_4... 2024-04-02T14:49:00.605 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-02T14:49:02.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:02 smithi122 ceph-mon[24801]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:02.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:02 smithi183 ceph-mon[27058]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:04.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:04 smithi122 ceph-mon[24801]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:04.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:04 smithi183 ceph-mon[27058]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:05.618 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:05 smithi122 ceph-mon[24801]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:05.618 INFO:teuthology.orchestra.run.smithi122.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-02T14:49:05.618 INFO:teuthology.orchestra.run.smithi122.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-02T14:49:05.618 INFO:teuthology.orchestra.run.smithi122.stderr: stderr: 10+0 records in 2024-04-02T14:49:05.618 INFO:teuthology.orchestra.run.smithi122.stderr:10+0 records out 2024-04-02T14:49:05.618 INFO:teuthology.orchestra.run.smithi122.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0110017 s, 953 MB/s 2024-04-02T14:49:05.618 INFO:teuthology.orchestra.run.smithi122.stderr:--> Zapping successful for: 2024-04-02T14:49:05.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:05 smithi183 ceph-mon[27058]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:08.344 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi122:vg_nvme/lv_4 2024-04-02T14:49:08.548 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:08 smithi122 ceph-mon[24801]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:08.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:08 smithi183 ceph-mon[27058]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:10.183 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:09 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:10.183 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:09 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:49:10.184 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:09 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:10.184 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:09 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:49:10.184 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:09 smithi122 ceph-mon[24801]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:10.184 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:09 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:10.198 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:09 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:10.199 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:09 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:49:10.199 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:09 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:10.199 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:09 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:49:10.199 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:09 smithi183 ceph-mon[27058]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:10.199 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:09 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:11.957 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:11 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:11.957 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:11 smithi122 ceph-mon[24801]: Updating smithi183:/etc/ceph/ceph.conf 2024-04-02T14:49:11.957 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:11 smithi122 ceph-mon[24801]: Updating smithi183:/etc/ceph/ceph.client.admin.keyring 2024-04-02T14:49:11.957 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:11 smithi122 ceph-mon[24801]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:11.958 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:11 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:11.958 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:11 smithi122 ceph-mon[24801]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi122:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:49:11.958 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:11 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:49:11.958 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:11 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:49:11.958 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:11 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:12.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:11 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:12.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:11 smithi183 ceph-mon[27058]: Updating smithi183:/etc/ceph/ceph.conf 2024-04-02T14:49:12.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:11 smithi183 ceph-mon[27058]: Updating smithi183:/etc/ceph/ceph.client.admin.keyring 2024-04-02T14:49:12.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:11 smithi183 ceph-mon[27058]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:12.152 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:11 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:12.152 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:11 smithi183 ceph-mon[27058]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi122:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:49:12.152 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:11 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:49:12.152 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:11 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:49:12.152 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:11 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:14.513 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:14 smithi122 ceph-mon[24801]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:14.514 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:14 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/1491479856' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4f45f72f-a976-4503-986c-e03c797072b9"}]: dispatch 2024-04-02T14:49:14.514 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:14 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/1491479856' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4f45f72f-a976-4503-986c-e03c797072b9"}]': finished 2024-04-02T14:49:14.514 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:14 smithi122 ceph-mon[24801]: osdmap e6: 1 total, 0 up, 1 in 2024-04-02T14:49:14.514 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:14 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:49:14.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:14 smithi183 ceph-mon[27058]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:14.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:14 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/1491479856' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4f45f72f-a976-4503-986c-e03c797072b9"}]: dispatch 2024-04-02T14:49:14.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:14 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/1491479856' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4f45f72f-a976-4503-986c-e03c797072b9"}]': finished 2024-04-02T14:49:14.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:14 smithi183 ceph-mon[27058]: osdmap e6: 1 total, 0 up, 1 in 2024-04-02T14:49:14.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:14 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:49:15.570 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:15 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/3851924829' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:49:15.570 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:15 smithi122 ceph-mon[24801]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:15.570 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:15 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:15.570 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:15 smithi122 ceph-mon[24801]: Updating smithi122:/etc/ceph/ceph.conf 2024-04-02T14:49:15.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:15 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/3851924829' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:49:15.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:15 smithi183 ceph-mon[27058]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:15.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:15 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:15.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:15 smithi183 ceph-mon[27058]: Updating smithi122:/etc/ceph/ceph.conf 2024-04-02T14:49:16.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:16 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:16.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:16 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:16.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:16 smithi122 ceph-mon[24801]: Reconfiguring mon.smithi122 (unknown last config time)... 2024-04-02T14:49:16.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:16 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T14:49:16.841 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:16 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T14:49:16.841 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:16 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:16.841 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:16 smithi122 ceph-mon[24801]: Reconfiguring daemon mon.smithi122 on smithi122 2024-04-02T14:49:16.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:16 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:16.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:16 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:16.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:16 smithi183 ceph-mon[27058]: Reconfiguring mon.smithi122 (unknown last config time)... 2024-04-02T14:49:16.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:16 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T14:49:16.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:16 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T14:49:16.902 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:16 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:16.902 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:16 smithi183 ceph-mon[27058]: Reconfiguring daemon mon.smithi122 on smithi122 2024-04-02T14:49:17.724 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:17 smithi122 ceph-mon[24801]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:17.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:17 smithi183 ceph-mon[27058]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:20.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:19 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:20.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:19 smithi122 ceph-mon[24801]: Reconfiguring mgr.smithi122.szjkfg (unknown last config time)... 2024-04-02T14:49:20.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:19 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi122.szjkfg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T14:49:20.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:19 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T14:49:20.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:19 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:20.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:19 smithi122 ceph-mon[24801]: Reconfiguring daemon mgr.smithi122.szjkfg on smithi122 2024-04-02T14:49:20.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:19 smithi122 ceph-mon[24801]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:20.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:19 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:20.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:19 smithi183 ceph-mon[27058]: Reconfiguring mgr.smithi122.szjkfg (unknown last config time)... 2024-04-02T14:49:20.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:19 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi122.szjkfg", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T14:49:20.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:19 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T14:49:20.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:19 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:20.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:19 smithi183 ceph-mon[27058]: Reconfiguring daemon mgr.smithi122.szjkfg on smithi122 2024-04-02T14:49:20.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:19 smithi183 ceph-mon[27058]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:22.319 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:22 smithi122 ceph-mon[24801]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:22.319 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:22 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:22.319 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:22 smithi122 ceph-mon[24801]: Reconfiguring alertmanager.smithi122 (dependencies changed)... 2024-04-02T14:49:22.319 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:22 smithi122 ceph-mon[24801]: Reconfiguring daemon alertmanager.smithi122 on smithi122 2024-04-02T14:49:22.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:22 smithi183 ceph-mon[27058]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:22.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:22 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:22.652 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:22 smithi183 ceph-mon[27058]: Reconfiguring alertmanager.smithi122 (dependencies changed)... 2024-04-02T14:49:22.652 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:22 smithi183 ceph-mon[27058]: Reconfiguring daemon alertmanager.smithi122 on smithi122 2024-04-02T14:49:23.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:23 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-02T14:49:23.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:23 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:23.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:23 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-02T14:49:23.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:23 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:24.432 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:24 smithi122 ceph-mon[24801]: Deploying daemon osd.0 on smithi122 2024-04-02T14:49:24.432 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:24 smithi122 ceph-mon[24801]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:24.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:24 smithi183 ceph-mon[27058]: Deploying daemon osd.0 on smithi122 2024-04-02T14:49:24.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:24 smithi183 ceph-mon[27058]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:25.775 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:25 smithi122 ceph-mon[24801]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:25.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:25 smithi183 ceph-mon[27058]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:28.255 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:28 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:28.256 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:28 smithi122 ceph-mon[24801]: Reconfiguring crash.smithi122 (monmap changed)... 2024-04-02T14:49:28.256 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:28 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi122", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-02T14:49:28.256 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:28 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:28.256 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:28 smithi122 ceph-mon[24801]: Reconfiguring daemon crash.smithi122 on smithi122 2024-04-02T14:49:28.256 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:28 smithi122 ceph-mon[24801]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:28.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:28 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:28.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:28 smithi183 ceph-mon[27058]: Reconfiguring crash.smithi122 (monmap changed)... 2024-04-02T14:49:28.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:28 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi122", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-02T14:49:28.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:28 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:28.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:28 smithi183 ceph-mon[27058]: Reconfiguring daemon crash.smithi122 on smithi122 2024-04-02T14:49:28.402 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:28 smithi183 ceph-mon[27058]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:29.480 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:49:29.480 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/trash_purge_schedule"}]: dispatch 2024-04-02T14:49:29.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:49:29.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/trash_purge_schedule"}]: dispatch 2024-04-02T14:49:30.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:30 smithi122 ceph-mon[24801]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:30.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:30 smithi183 ceph-mon[27058]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:32.587 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:32 smithi122 ceph-mon[24801]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:32.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:32.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:32 smithi183 ceph-mon[27058]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:32.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:32 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:34.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:34 smithi122 ceph-mon[24801]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:34.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:34 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:34.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:34 smithi183 ceph-mon[27058]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:34.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:34 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:35.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:35 smithi122 ceph-mon[24801]: Reconfiguring grafana.smithi122 (dependencies changed)... 2024-04-02T14:49:35.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:35 smithi122 ceph-mon[24801]: Reconfiguring daemon grafana.smithi122 on smithi122 2024-04-02T14:49:35.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:35 smithi122 ceph-mon[24801]: from='osd.0 [v2:172.21.15.122:6802/418135664,v1:172.21.15.122:6803/418135664]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T14:49:35.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:35 smithi183 ceph-mon[27058]: Reconfiguring grafana.smithi122 (dependencies changed)... 2024-04-02T14:49:35.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:35 smithi183 ceph-mon[27058]: Reconfiguring daemon grafana.smithi122 on smithi122 2024-04-02T14:49:35.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:35 smithi183 ceph-mon[27058]: from='osd.0 [v2:172.21.15.122:6802/418135664,v1:172.21.15.122:6803/418135664]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T14:49:36.523 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:36 smithi122 ceph-mon[24801]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:36.523 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:36 smithi122 ceph-mon[24801]: from='osd.0 [v2:172.21.15.122:6802/418135664,v1:172.21.15.122:6803/418135664]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-02T14:49:36.523 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:36 smithi122 ceph-mon[24801]: osdmap e7: 1 total, 0 up, 1 in 2024-04-02T14:49:36.523 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:36 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:49:36.523 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:36 smithi122 ceph-mon[24801]: from='osd.0 [v2:172.21.15.122:6802/418135664,v1:172.21.15.122:6803/418135664]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi122", "root=default"]}]: dispatch 2024-04-02T14:49:36.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:36 smithi183 ceph-mon[27058]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:36.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:36 smithi183 ceph-mon[27058]: from='osd.0 [v2:172.21.15.122:6802/418135664,v1:172.21.15.122:6803/418135664]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-02T14:49:36.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:36 smithi183 ceph-mon[27058]: osdmap e7: 1 total, 0 up, 1 in 2024-04-02T14:49:36.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:36 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:49:36.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:36 smithi183 ceph-mon[27058]: from='osd.0 [v2:172.21.15.122:6802/418135664,v1:172.21.15.122:6803/418135664]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi122", "root=default"]}]: dispatch 2024-04-02T14:49:37.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:37 smithi122 ceph-mon[24801]: from='osd.0 [v2:172.21.15.122:6802/418135664,v1:172.21.15.122:6803/418135664]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi122", "root=default"]}]': finished 2024-04-02T14:49:37.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:37 smithi122 ceph-mon[24801]: osdmap e8: 1 total, 0 up, 1 in 2024-04-02T14:49:37.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:37 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:49:37.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:37 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:49:37.841 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:37 smithi122 ceph-mon[24801]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:37.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:37 smithi183 ceph-mon[27058]: from='osd.0 [v2:172.21.15.122:6802/418135664,v1:172.21.15.122:6803/418135664]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi122", "root=default"]}]': finished 2024-04-02T14:49:37.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:37 smithi183 ceph-mon[27058]: osdmap e8: 1 total, 0 up, 1 in 2024-04-02T14:49:37.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:37 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:49:37.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:37 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:49:37.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:37 smithi183 ceph-mon[27058]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T14:49:38.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:38 smithi122 ceph-mon[24801]: purged_snaps scrub starts 2024-04-02T14:49:38.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:38 smithi122 ceph-mon[24801]: purged_snaps scrub ok 2024-04-02T14:49:38.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:38 smithi122 ceph-mon[24801]: osd.0 [v2:172.21.15.122:6802/418135664,v1:172.21.15.122:6803/418135664] boot 2024-04-02T14:49:38.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:38 smithi122 ceph-mon[24801]: osdmap e9: 1 total, 1 up, 1 in 2024-04-02T14:49:38.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:38 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:49:38.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:38 smithi183 ceph-mon[27058]: purged_snaps scrub starts 2024-04-02T14:49:38.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:38 smithi183 ceph-mon[27058]: purged_snaps scrub ok 2024-04-02T14:49:38.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:38 smithi183 ceph-mon[27058]: osd.0 [v2:172.21.15.122:6802/418135664,v1:172.21.15.122:6803/418135664] boot 2024-04-02T14:49:38.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:38 smithi183 ceph-mon[27058]: osdmap e9: 1 total, 1 up, 1 in 2024-04-02T14:49:38.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:38 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:49:39.783 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:39 smithi122 ceph-mon[24801]: osdmap e10: 1 total, 1 up, 1 in 2024-04-02T14:49:39.783 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:39 smithi122 ceph-mon[24801]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:39.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:39 smithi183 ceph-mon[27058]: osdmap e10: 1 total, 1 up, 1 in 2024-04-02T14:49:39.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:39 smithi183 ceph-mon[27058]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:40.912 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:40 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:40.912 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:40 smithi122 ceph-mon[24801]: Reconfiguring crash.smithi183 (monmap changed)... 2024-04-02T14:49:40.912 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:40 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi183", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-02T14:49:40.912 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:40 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:40.912 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:40 smithi122 ceph-mon[24801]: Reconfiguring daemon crash.smithi183 on smithi183 2024-04-02T14:49:41.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:40 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:41.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:40 smithi183 ceph-mon[27058]: Reconfiguring crash.smithi183 (monmap changed)... 2024-04-02T14:49:41.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:40 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi183", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-02T14:49:41.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:40 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:41.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:40 smithi183 ceph-mon[27058]: Reconfiguring daemon crash.smithi183 on smithi183 2024-04-02T14:49:41.260 INFO:teuthology.orchestra.run.smithi122.stdout:Created osd(s) 0 on host 'smithi122' 2024-04-02T14:49:41.913 DEBUG:teuthology.orchestra.run.smithi122:osd.0> sudo journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.0.service 2024-04-02T14:49:41.915 INFO:tasks.cephadm:Deploying osd.1 on smithi122 with /dev/vg_nvme/lv_3... 2024-04-02T14:49:41.915 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-02T14:49:42.469 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:42 smithi122 ceph-mon[24801]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:42.469 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:42 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:42.469 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:42 smithi122 ceph-mon[24801]: Reconfiguring mgr.smithi183.smxedq (monmap changed)... 2024-04-02T14:49:42.469 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:42 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi183.smxedq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T14:49:42.469 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:42 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T14:49:42.469 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:42 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:42.470 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:42 smithi122 ceph-mon[24801]: Reconfiguring daemon mgr.smithi183.smxedq on smithi183 2024-04-02T14:49:42.470 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:42 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:42.503 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:42 smithi183 ceph-mon[27058]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:42.503 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:42 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:42.503 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:42 smithi183 ceph-mon[27058]: Reconfiguring mgr.smithi183.smxedq (monmap changed)... 2024-04-02T14:49:42.503 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:42 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi183.smxedq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T14:49:42.504 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:42 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T14:49:42.504 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:42 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:42.504 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:42 smithi183 ceph-mon[27058]: Reconfiguring daemon mgr.smithi183.smxedq on smithi183 2024-04-02T14:49:42.504 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:42 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:43.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:43.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:43 smithi122 ceph-mon[24801]: Reconfiguring mon.smithi183 (monmap changed)... 2024-04-02T14:49:43.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T14:49:43.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T14:49:43.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:43.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:43 smithi122 ceph-mon[24801]: Reconfiguring daemon mon.smithi183 on smithi183 2024-04-02T14:49:43.841 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:43 smithi122 ceph-mon[24801]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:43.841 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:43.842 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:43 smithi183 ceph-mon[27058]: Reconfiguring mon.smithi183 (monmap changed)... 2024-04-02T14:49:43.842 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T14:49:43.842 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T14:49:43.842 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:43.842 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:43 smithi183 ceph-mon[27058]: Reconfiguring daemon mon.smithi183 on smithi183 2024-04-02T14:49:43.842 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:43 smithi183 ceph-mon[27058]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:43.982 INFO:teuthology.orchestra.run.smithi122.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-02T14:49:43.983 INFO:teuthology.orchestra.run.smithi122.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-02T14:49:43.983 INFO:teuthology.orchestra.run.smithi122.stderr: stderr: 10+0 records in 2024-04-02T14:49:43.983 INFO:teuthology.orchestra.run.smithi122.stderr:10+0 records out 2024-04-02T14:49:43.983 INFO:teuthology.orchestra.run.smithi122.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0137526 s, 762 MB/s 2024-04-02T14:49:43.983 INFO:teuthology.orchestra.run.smithi122.stderr:--> Zapping successful for: 2024-04-02T14:49:44.670 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi122:vg_nvme/lv_3 2024-04-02T14:49:45.026 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:45.027 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T14:49:45.027 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:44 smithi122 ceph-mon[24801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T14:49:45.027 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.122:9093"}]: dispatch 2024-04-02T14:49:45.027 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:44 smithi122 ceph-mon[24801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.122:9093"}]: dispatch 2024-04-02T14:49:45.027 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:45.027 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T14:49:45.027 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:44 smithi122 ceph-mon[24801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T14:49:45.027 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.122:3000"}]: dispatch 2024-04-02T14:49:45.027 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:44 smithi122 ceph-mon[24801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.122:3000"}]: dispatch 2024-04-02T14:49:45.027 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:45.027 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T14:49:45.027 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:44 smithi122 ceph-mon[24801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T14:49:45.027 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.122:9095"}]: dispatch 2024-04-02T14:49:45.027 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:44 smithi122 ceph-mon[24801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.122:9095"}]: dispatch 2024-04-02T14:49:45.028 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:45.028 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:49:45.028 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:45.028 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:49:45.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:44 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:45.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:44 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T14:49:45.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:44 smithi183 ceph-mon[27058]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T14:49:45.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:44 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.122:9093"}]: dispatch 2024-04-02T14:49:45.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:44 smithi183 ceph-mon[27058]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.122:9093"}]: dispatch 2024-04-02T14:49:45.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:44 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:45.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:44 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T14:49:45.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:44 smithi183 ceph-mon[27058]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T14:49:45.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:44 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.122:3000"}]: dispatch 2024-04-02T14:49:45.152 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:44 smithi183 ceph-mon[27058]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.122:3000"}]: dispatch 2024-04-02T14:49:45.152 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:44 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:45.152 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:44 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T14:49:45.152 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:44 smithi183 ceph-mon[27058]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T14:49:45.152 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:44 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.122:9095"}]: dispatch 2024-04-02T14:49:45.152 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:44 smithi183 ceph-mon[27058]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.122:9095"}]: dispatch 2024-04-02T14:49:45.152 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:44 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:45.152 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:44 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:49:45.152 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:44 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:45.152 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:44 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:49:46.283 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:45 smithi122 ceph-mon[24801]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:46.284 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:45 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:46.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:45 smithi183 ceph-mon[27058]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:46.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:45 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:48.524 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:48 smithi122 ceph-mon[24801]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:48.524 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:48 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:49:48.524 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:48 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:49:48.524 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:48 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:48.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:48 smithi183 ceph-mon[27058]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:48.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:48 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:49:48.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:48 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:49:48.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:48 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:49.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:49 smithi122 ceph-mon[24801]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi122:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:49:49.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:49 smithi183 ceph-mon[27058]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi122:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:49:50.489 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:50 smithi122 ceph-mon[24801]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:50.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:50 smithi183 ceph-mon[27058]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:52.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:51 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:52.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:51 smithi122 ceph-mon[24801]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:52.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:51 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/51532883' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "796f1929-36a0-43ae-8fc9-a02fcb40699a"}]: dispatch 2024-04-02T14:49:52.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:51 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/51532883' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "796f1929-36a0-43ae-8fc9-a02fcb40699a"}]': finished 2024-04-02T14:49:52.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:51 smithi122 ceph-mon[24801]: osdmap e11: 2 total, 1 up, 2 in 2024-04-02T14:49:52.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:51 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:49:52.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:51 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:52.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:51 smithi183 ceph-mon[27058]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:52.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:51 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/51532883' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "796f1929-36a0-43ae-8fc9-a02fcb40699a"}]: dispatch 2024-04-02T14:49:52.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:51 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/51532883' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "796f1929-36a0-43ae-8fc9-a02fcb40699a"}]': finished 2024-04-02T14:49:52.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:51 smithi183 ceph-mon[27058]: osdmap e11: 2 total, 1 up, 2 in 2024-04-02T14:49:52.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:51 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:49:53.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:52 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/2972375887' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:49:53.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:52 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/2972375887' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:49:54.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:53 smithi122 ceph-mon[24801]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:54.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:53 smithi183 ceph-mon[27058]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:55.737 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:55 smithi122 ceph-mon[24801]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:55.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:55 smithi183 ceph-mon[27058]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:58.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:57 smithi183 ceph-mon[27058]: Detected new or changed devices on smithi122 2024-04-02T14:49:58.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:57 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:58.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:57 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:49:58.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:57 smithi183 ceph-mon[27058]: Adjusting osd_memory_target on smithi122 to 12864M 2024-04-02T14:49:58.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:57 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:58.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:57 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:58.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:57 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:49:58.152 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:57 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:58.152 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:57 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:49:58.152 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:57 smithi183 ceph-mon[27058]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:58.320 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:57 smithi122 ceph-mon[24801]: Detected new or changed devices on smithi122 2024-04-02T14:49:58.320 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:57 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:58.320 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:57 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:49:58.320 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:57 smithi122 ceph-mon[24801]: Adjusting osd_memory_target on smithi122 to 12864M 2024-04-02T14:49:58.320 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:57 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:58.320 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:57 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:58.321 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:57 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:49:58.321 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:57 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:58.321 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:57 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:49:58.321 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:57 smithi122 ceph-mon[24801]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:59.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:59 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:59.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:59 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-02T14:49:59.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:59 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:59.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:59 smithi122 ceph-mon[24801]: Deploying daemon osd.1 on smithi122 2024-04-02T14:49:59.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:49:59 smithi122 ceph-mon[24801]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:49:59.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:59 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:49:59.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:59 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-02T14:49:59.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:59 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:49:59.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:59 smithi183 ceph-mon[27058]: Deploying daemon osd.1 on smithi122 2024-04-02T14:49:59.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:49:59 smithi183 ceph-mon[27058]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:50:00.834 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:00 smithi122 ceph-mon[24801]: overall HEALTH_OK 2024-04-02T14:50:00.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:00 smithi183 ceph-mon[27058]: overall HEALTH_OK 2024-04-02T14:50:01.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:01 smithi122 ceph-mon[24801]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:50:01.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:01 smithi183 ceph-mon[27058]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:50:04.251 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:04 smithi122 ceph-mon[24801]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:50:04.251 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:04 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:04.252 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:04 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:04.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:04 smithi183 ceph-mon[27058]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:50:04.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:04 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:04.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:04 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:05.664 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:05 smithi122 ceph-mon[24801]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:50:05.862 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:05 smithi183 ceph-mon[27058]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:50:07.089 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:06 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:07.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:06 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:50:07.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:06 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:50:07.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:06 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:50:07.150 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:06 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:07.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:06 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:50:07.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:06 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:50:07.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:06 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:50:08.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:07 smithi122 ceph-mon[24801]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:50:08.091 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:07 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:08.150 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:07 smithi183 ceph-mon[27058]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:50:08.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:07 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:08.548 INFO:teuthology.orchestra.run.smithi122.stdout:Created osd(s) 1 on host 'smithi122' 2024-04-02T14:50:09.335 DEBUG:teuthology.orchestra.run.smithi122:osd.1> sudo journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.1.service 2024-04-02T14:50:09.338 INFO:tasks.cephadm:Deploying osd.2 on smithi122 with /dev/vg_nvme/lv_2... 2024-04-02T14:50:09.338 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-02T14:50:09.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:09 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:09.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:09 smithi122 ceph-mon[24801]: from='osd.1 [v2:172.21.15.122:6810/2295878653,v1:172.21.15.122:6811/2295878653]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T14:50:09.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:09 smithi122 ceph-mon[24801]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:50:09.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:09 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:09.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:09 smithi183 ceph-mon[27058]: from='osd.1 [v2:172.21.15.122:6810/2295878653,v1:172.21.15.122:6811/2295878653]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T14:50:09.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:09 smithi183 ceph-mon[27058]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:50:10.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:10 smithi122 ceph-mon[24801]: from='osd.1 [v2:172.21.15.122:6810/2295878653,v1:172.21.15.122:6811/2295878653]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-02T14:50:10.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:10 smithi122 ceph-mon[24801]: osdmap e12: 2 total, 1 up, 2 in 2024-04-02T14:50:10.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:10 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:50:10.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:10 smithi122 ceph-mon[24801]: from='osd.1 [v2:172.21.15.122:6810/2295878653,v1:172.21.15.122:6811/2295878653]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi122", "root=default"]}]: dispatch 2024-04-02T14:50:10.840 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:50:10 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-1[39204]: 2024-04-02T14:50:10.565+0000 7f2315f03700 -1 osd.1 0 waiting for initial osdmap 2024-04-02T14:50:10.840 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:50:10 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-1[39204]: 2024-04-02T14:50:10.568+0000 7f230d093700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T14:50:10.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:10 smithi183 ceph-mon[27058]: from='osd.1 [v2:172.21.15.122:6810/2295878653,v1:172.21.15.122:6811/2295878653]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-02T14:50:10.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:10 smithi183 ceph-mon[27058]: osdmap e12: 2 total, 1 up, 2 in 2024-04-02T14:50:10.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:10 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:50:10.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:10 smithi183 ceph-mon[27058]: from='osd.1 [v2:172.21.15.122:6810/2295878653,v1:172.21.15.122:6811/2295878653]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi122", "root=default"]}]: dispatch 2024-04-02T14:50:11.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:11 smithi122 ceph-mon[24801]: from='osd.1 [v2:172.21.15.122:6810/2295878653,v1:172.21.15.122:6811/2295878653]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi122", "root=default"]}]': finished 2024-04-02T14:50:11.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:11 smithi122 ceph-mon[24801]: osdmap e13: 2 total, 1 up, 2 in 2024-04-02T14:50:11.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:11 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:50:11.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:11 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:50:11.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:11 smithi122 ceph-mon[24801]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:50:11.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:11 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:50:11.883 INFO:teuthology.orchestra.run.smithi122.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-02T14:50:11.884 INFO:teuthology.orchestra.run.smithi122.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-02T14:50:11.884 INFO:teuthology.orchestra.run.smithi122.stderr: stderr: 10+0 records in 2024-04-02T14:50:11.884 INFO:teuthology.orchestra.run.smithi122.stderr:10+0 records out 2024-04-02T14:50:11.884 INFO:teuthology.orchestra.run.smithi122.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0120913 s, 867 MB/s 2024-04-02T14:50:11.884 INFO:teuthology.orchestra.run.smithi122.stderr:--> Zapping successful for: 2024-04-02T14:50:11.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:11 smithi183 ceph-mon[27058]: from='osd.1 [v2:172.21.15.122:6810/2295878653,v1:172.21.15.122:6811/2295878653]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi122", "root=default"]}]': finished 2024-04-02T14:50:11.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:11 smithi183 ceph-mon[27058]: osdmap e13: 2 total, 1 up, 2 in 2024-04-02T14:50:11.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:11 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:50:11.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:11 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:50:11.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:11 smithi183 ceph-mon[27058]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T14:50:11.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:11 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:50:12.567 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi122:vg_nvme/lv_2 2024-04-02T14:50:12.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:12 smithi122 ceph-mon[24801]: purged_snaps scrub starts 2024-04-02T14:50:12.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:12 smithi122 ceph-mon[24801]: purged_snaps scrub ok 2024-04-02T14:50:12.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:12 smithi122 ceph-mon[24801]: osd.1 [v2:172.21.15.122:6810/2295878653,v1:172.21.15.122:6811/2295878653] boot 2024-04-02T14:50:12.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:12 smithi122 ceph-mon[24801]: osdmap e14: 2 total, 2 up, 2 in 2024-04-02T14:50:12.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:12 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:50:12.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:12 smithi183 ceph-mon[27058]: purged_snaps scrub starts 2024-04-02T14:50:12.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:12 smithi183 ceph-mon[27058]: purged_snaps scrub ok 2024-04-02T14:50:12.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:12 smithi183 ceph-mon[27058]: osd.1 [v2:172.21.15.122:6810/2295878653,v1:172.21.15.122:6811/2295878653] boot 2024-04-02T14:50:12.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:12 smithi183 ceph-mon[27058]: osdmap e14: 2 total, 2 up, 2 in 2024-04-02T14:50:12.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:12 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:50:13.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:13 smithi122 ceph-mon[24801]: osdmap e15: 2 total, 2 up, 2 in 2024-04-02T14:50:13.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:13 smithi122 ceph-mon[24801]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:13.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:13 smithi183 ceph-mon[27058]: osdmap e15: 2 total, 2 up, 2 in 2024-04-02T14:50:13.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:13 smithi183 ceph-mon[27058]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:15.752 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:15 smithi122 ceph-mon[24801]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:15.753 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:15 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:50:15.753 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:15 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:50:15.753 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:15 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:50:15.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:15 smithi183 ceph-mon[27058]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:15.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:15 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:50:15.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:15 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:50:15.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:15 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:50:17.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:17 smithi122 ceph-mon[24801]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi122:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:50:17.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:17 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:17.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:17 smithi183 ceph-mon[27058]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi122:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:50:17.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:17 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:18.346 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:18 smithi122 ceph-mon[24801]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:18.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:18 smithi183 ceph-mon[27058]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:19.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:19 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/3428419630' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "369ff431-5778-4d3a-9c18-c4e34b14a540"}]: dispatch 2024-04-02T14:50:19.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:19 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/3428419630' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "369ff431-5778-4d3a-9c18-c4e34b14a540"}]': finished 2024-04-02T14:50:19.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:19 smithi183 ceph-mon[27058]: osdmap e16: 3 total, 2 up, 3 in 2024-04-02T14:50:19.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:19 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:50:19.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:19 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/3428419630' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "369ff431-5778-4d3a-9c18-c4e34b14a540"}]: dispatch 2024-04-02T14:50:19.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:19 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/3428419630' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "369ff431-5778-4d3a-9c18-c4e34b14a540"}]': finished 2024-04-02T14:50:19.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:19 smithi122 ceph-mon[24801]: osdmap e16: 3 total, 2 up, 3 in 2024-04-02T14:50:19.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:19 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:50:20.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:20 smithi183 ceph-mon[27058]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:20.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:20 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/2024696690' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:50:20.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:20 smithi122 ceph-mon[24801]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:20.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:20 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/2024696690' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:50:22.503 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:22 smithi122 ceph-mon[24801]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:22.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:22 smithi183 ceph-mon[27058]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:24.510 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:24 smithi122 ceph-mon[24801]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:24.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:24 smithi183 ceph-mon[27058]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:26.108 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:25 smithi122 ceph-mon[24801]: Detected new or changed devices on smithi122 2024-04-02T14:50:26.109 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:25 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:26.109 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:25 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:50:26.109 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:25 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:50:26.109 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:25 smithi122 ceph-mon[24801]: Adjusting osd_memory_target on smithi122 to 6432M 2024-04-02T14:50:26.109 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:25 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:26.109 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:25 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:26.109 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:25 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:50:26.109 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:25 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:50:26.109 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:25 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:50:26.109 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:25 smithi122 ceph-mon[24801]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:26.150 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:25 smithi183 ceph-mon[27058]: Detected new or changed devices on smithi122 2024-04-02T14:50:26.150 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:25 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:26.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:25 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:50:26.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:25 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:50:26.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:25 smithi183 ceph-mon[27058]: Adjusting osd_memory_target on smithi122 to 6432M 2024-04-02T14:50:26.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:25 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:26.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:25 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:26.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:25 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:50:26.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:25 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:50:26.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:25 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:50:26.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:25 smithi183 ceph-mon[27058]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:27.150 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:26 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-02T14:50:27.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:26 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:50:27.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:26 smithi183 ceph-mon[27058]: Deploying daemon osd.2 on smithi122 2024-04-02T14:50:27.156 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:26 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-02T14:50:27.156 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:26 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:50:27.156 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:26 smithi122 ceph-mon[24801]: Deploying daemon osd.2 on smithi122 2024-04-02T14:50:28.150 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:27 smithi183 ceph-mon[27058]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:28.327 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:27 smithi122 ceph-mon[24801]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:29.512 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:50:29.513 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/trash_purge_schedule"}]: dispatch 2024-04-02T14:50:29.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:50:29.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/trash_purge_schedule"}]: dispatch 2024-04-02T14:50:30.339 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:30 smithi122 ceph-mon[24801]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:30.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:30 smithi183 ceph-mon[27058]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:32.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:32 smithi122 ceph-mon[24801]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:32.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:32.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:32.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:32 smithi183 ceph-mon[27058]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:32.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:32 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:32.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:32 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:34.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:34 smithi122 ceph-mon[24801]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:34.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:34 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:34.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:34 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:50:34.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:34 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:50:34.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:34 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:50:34.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:34 smithi183 ceph-mon[27058]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:34.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:34 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:34.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:34 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:50:34.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:34 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:50:34.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:34 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:50:35.682 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:35 smithi122 ceph-mon[24801]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:35.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:35 smithi183 ceph-mon[27058]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:37.240 INFO:teuthology.orchestra.run.smithi122.stdout:Created osd(s) 2 on host 'smithi122' 2024-04-02T14:50:37.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:37 smithi122 ceph-mon[24801]: from='osd.2 [v2:172.21.15.122:6818/1200971619,v1:172.21.15.122:6819/1200971619]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T14:50:37.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:37 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:37.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:37 smithi183 ceph-mon[27058]: from='osd.2 [v2:172.21.15.122:6818/1200971619,v1:172.21.15.122:6819/1200971619]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T14:50:37.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:37 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:38.096 DEBUG:teuthology.orchestra.run.smithi122:osd.2> sudo journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.2.service 2024-04-02T14:50:38.098 INFO:tasks.cephadm:Deploying osd.3 on smithi122 with /dev/vg_nvme/lv_1... 2024-04-02T14:50:38.098 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-02T14:50:38.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:38 smithi122 ceph-mon[24801]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:38.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:38 smithi122 ceph-mon[24801]: from='osd.2 [v2:172.21.15.122:6818/1200971619,v1:172.21.15.122:6819/1200971619]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-02T14:50:38.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:38 smithi122 ceph-mon[24801]: osdmap e17: 3 total, 2 up, 3 in 2024-04-02T14:50:38.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:38 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:50:38.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:38 smithi122 ceph-mon[24801]: from='osd.2 [v2:172.21.15.122:6818/1200971619,v1:172.21.15.122:6819/1200971619]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi122", "root=default"]}]: dispatch 2024-04-02T14:50:38.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:38 smithi122 ceph-mon[24801]: from='osd.2 [v2:172.21.15.122:6818/1200971619,v1:172.21.15.122:6819/1200971619]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi122", "root=default"]}]': finished 2024-04-02T14:50:38.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:38 smithi122 ceph-mon[24801]: osdmap e18: 3 total, 2 up, 3 in 2024-04-02T14:50:38.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:38 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:50:38.340 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:50:38 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-2[42918]: 2024-04-02T14:50:38.281+0000 7f20590af700 -1 osd.2 0 waiting for initial osdmap 2024-04-02T14:50:38.341 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:50:38 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-2[42918]: 2024-04-02T14:50:38.284+0000 7f2053a46700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T14:50:38.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:38 smithi183 ceph-mon[27058]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:38.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:38 smithi183 ceph-mon[27058]: from='osd.2 [v2:172.21.15.122:6818/1200971619,v1:172.21.15.122:6819/1200971619]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-02T14:50:38.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:38 smithi183 ceph-mon[27058]: osdmap e17: 3 total, 2 up, 3 in 2024-04-02T14:50:38.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:38 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:50:38.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:38 smithi183 ceph-mon[27058]: from='osd.2 [v2:172.21.15.122:6818/1200971619,v1:172.21.15.122:6819/1200971619]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi122", "root=default"]}]: dispatch 2024-04-02T14:50:38.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:38 smithi183 ceph-mon[27058]: from='osd.2 [v2:172.21.15.122:6818/1200971619,v1:172.21.15.122:6819/1200971619]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi122", "root=default"]}]': finished 2024-04-02T14:50:38.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:38 smithi183 ceph-mon[27058]: osdmap e18: 3 total, 2 up, 3 in 2024-04-02T14:50:38.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:38 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:50:39.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:39 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:50:39.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:39 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:50:39.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:39 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:50:39.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:39 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:50:40.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:40 smithi122 ceph-mon[24801]: purged_snaps scrub starts 2024-04-02T14:50:40.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:40 smithi122 ceph-mon[24801]: purged_snaps scrub ok 2024-04-02T14:50:40.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:40 smithi122 ceph-mon[24801]: pgmap v71: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:40.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:40 smithi122 ceph-mon[24801]: osd.2 [v2:172.21.15.122:6818/1200971619,v1:172.21.15.122:6819/1200971619] boot 2024-04-02T14:50:40.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:40 smithi122 ceph-mon[24801]: osdmap e19: 3 total, 3 up, 3 in 2024-04-02T14:50:40.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:40 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:50:40.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:40 smithi183 ceph-mon[27058]: purged_snaps scrub starts 2024-04-02T14:50:40.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:40 smithi183 ceph-mon[27058]: purged_snaps scrub ok 2024-04-02T14:50:40.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:40 smithi183 ceph-mon[27058]: pgmap v71: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-04-02T14:50:40.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:40 smithi183 ceph-mon[27058]: osd.2 [v2:172.21.15.122:6818/1200971619,v1:172.21.15.122:6819/1200971619] boot 2024-04-02T14:50:40.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:40 smithi183 ceph-mon[27058]: osdmap e19: 3 total, 3 up, 3 in 2024-04-02T14:50:40.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:40 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:50:40.719 INFO:teuthology.orchestra.run.smithi122.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-02T14:50:40.719 INFO:teuthology.orchestra.run.smithi122.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-02T14:50:40.720 INFO:teuthology.orchestra.run.smithi122.stderr: stderr: 10+0 records in 2024-04-02T14:50:40.720 INFO:teuthology.orchestra.run.smithi122.stderr:10+0 records out 2024-04-02T14:50:40.720 INFO:teuthology.orchestra.run.smithi122.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0124806 s, 840 MB/s 2024-04-02T14:50:40.720 INFO:teuthology.orchestra.run.smithi122.stderr:--> Zapping successful for: 2024-04-02T14:50:41.348 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi122:vg_nvme/lv_1 2024-04-02T14:50:41.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:41 smithi122 ceph-mon[24801]: osdmap e20: 3 total, 3 up, 3 in 2024-04-02T14:50:41.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:41 smithi122 ceph-mon[24801]: pgmap v74: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:50:41.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:41 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-02T14:50:41.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:41 smithi183 ceph-mon[27058]: osdmap e20: 3 total, 3 up, 3 in 2024-04-02T14:50:41.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:41 smithi183 ceph-mon[27058]: pgmap v74: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:50:41.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:41 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-02T14:50:42.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:42 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-02T14:50:42.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:42 smithi183 ceph-mon[27058]: osdmap e21: 3 total, 3 up, 3 in 2024-04-02T14:50:42.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:42 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-02T14:50:42.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:42 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-02T14:50:42.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:42 smithi122 ceph-mon[24801]: osdmap e21: 3 total, 3 up, 3 in 2024-04-02T14:50:42.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:42 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-02T14:50:43.742 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-02T14:50:43.743 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:43 smithi122 ceph-mon[24801]: osdmap e22: 3 total, 3 up, 3 in 2024-04-02T14:50:43.743 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:43 smithi122 ceph-mon[24801]: pgmap v77: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:50:43.743 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:50:43 smithi122 sudo[44282]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T14:50:43.743 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:50:43 smithi122 sudo[44282]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:50:43.743 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:50:43 smithi122 sudo[44282]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:50:43.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-02T14:50:43.964 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:43 smithi183 ceph-mon[27058]: osdmap e22: 3 total, 3 up, 3 in 2024-04-02T14:50:43.965 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:43 smithi183 ceph-mon[27058]: pgmap v77: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:50:44.017 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:50:43 smithi122 sudo[44293]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T14:50:44.017 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:50:43 smithi122 sudo[44293]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:50:44.018 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:50:43 smithi122 sudo[44293]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:50:44.284 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:50:44 smithi122 sudo[44296]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T14:50:44.284 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:50:44 smithi122 sudo[44296]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:50:44.285 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:50:44 smithi122 sudo[44296]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:50:44.540 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:50:44 smithi122 sudo[44356]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T14:50:44.540 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:50:44 smithi122 sudo[44356]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:50:44.540 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:50:44 smithi122 sudo[44356]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:50:44.811 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:50:44 smithi122 sudo[44433]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T14:50:44.811 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:50:44 smithi122 sudo[44433]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:50:44.811 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:50:44 smithi122 sudo[44433]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:50:44.811 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:44 smithi122 ceph-mon[24801]: osdmap e23: 3 total, 3 up, 3 in 2024-04-02T14:50:44.811 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:44.811 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:44.811 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:50:44.811 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:50:44.811 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:44 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:50:44.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:44 smithi183 ceph-mon[27058]: osdmap e23: 3 total, 3 up, 3 in 2024-04-02T14:50:44.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:44 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:44.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:44 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:44.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:44 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:50:44.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:44 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:50:44.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:44 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:50:45.074 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:50:44 smithi122 sudo[44507]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T14:50:45.074 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:50:44 smithi122 sudo[44507]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:50:45.074 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:50:44 smithi122 sudo[44507]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:50:45.330 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:45 smithi122 sudo[44528]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T14:50:45.330 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:45 smithi122 sudo[44528]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:50:45.331 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:45 smithi122 sudo[44528]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:50:45.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:45 smithi122 sudo[44544]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T14:50:45.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:45 smithi122 sudo[44544]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:50:45.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:45 smithi122 sudo[44544]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:50:45.830 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:45 smithi183 sudo[29374]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T14:50:45.831 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:45 smithi183 sudo[29374]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:50:45.831 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:45 smithi183 sudo[29374]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:50:45.831 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:45 smithi183 sudo[29377]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T14:50:45.831 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:45 smithi183 sudo[29377]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T14:50:45.831 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:45 smithi183 sudo[29377]: pam_unix(sudo:session): session closed for user root 2024-04-02T14:50:46.089 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:45 smithi122 ceph-mon[24801]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi122:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:50:46.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:45 smithi122 ceph-mon[24801]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T14:50:46.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:45 smithi122 ceph-mon[24801]: pgmap v79: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:50:46.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:45 smithi122 ceph-mon[24801]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T14:50:46.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:45 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi122"}]: dispatch 2024-04-02T14:50:46.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:45 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi183"}]: dispatch 2024-04-02T14:50:46.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:45 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi122"}]: dispatch 2024-04-02T14:50:46.090 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:45 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi183"}]: dispatch 2024-04-02T14:50:46.150 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:45 smithi183 ceph-mon[27058]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi122:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:50:46.150 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:45 smithi183 ceph-mon[27058]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T14:50:46.150 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:45 smithi183 ceph-mon[27058]: pgmap v79: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:50:46.150 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:45 smithi183 ceph-mon[27058]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T14:50:46.150 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:45 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi122"}]: dispatch 2024-04-02T14:50:46.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:45 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi183"}]: dispatch 2024-04-02T14:50:46.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:45 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi122"}]: dispatch 2024-04-02T14:50:46.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:45 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mon metadata", "id": "smithi183"}]: dispatch 2024-04-02T14:50:46.925 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:46 smithi122 ceph-mon[24801]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T14:50:46.925 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:46 smithi122 ceph-mon[24801]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T14:50:46.925 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:46 smithi122 ceph-mon[24801]: mgrmap e19: smithi122.szjkfg(active, since 2m), standbys: smithi183.smxedq 2024-04-02T14:50:46.926 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:46 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:47.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:46 smithi183 ceph-mon[27058]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T14:50:47.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:46 smithi183 ceph-mon[27058]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T14:50:47.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:46 smithi183 ceph-mon[27058]: mgrmap e19: smithi122.szjkfg(active, since 2m), standbys: smithi183.smxedq 2024-04-02T14:50:47.151 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:46 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:48.089 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:47 smithi122 ceph-mon[24801]: pgmap v80: 1 pgs: 1 creating+activating; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:50:48.150 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:47 smithi183 ceph-mon[27058]: pgmap v80: 1 pgs: 1 creating+activating; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:50:49.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:48 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:49.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:48 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:50:49.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:48 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:50:49.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:48 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:50:49.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:48 smithi122 ceph-mon[24801]: Adjusting osd_memory_target on smithi122 to 4288M 2024-04-02T14:50:49.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:48 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:49.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:48 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:49.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:48 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:50:49.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:48 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:50:49.340 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:48 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:50:49.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:48 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:49.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:48 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/1058958620' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "135c10c3-46ab-48b1-967e-fdcfc72eaaa4"}]: dispatch 2024-04-02T14:50:49.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:48 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/1058958620' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "135c10c3-46ab-48b1-967e-fdcfc72eaaa4"}]': finished 2024-04-02T14:50:49.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:48 smithi122 ceph-mon[24801]: osdmap e24: 4 total, 3 up, 4 in 2024-04-02T14:50:49.341 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:48 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:50:49.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:48 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:49.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:48 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:50:49.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:48 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:50:49.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:48 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:50:49.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:48 smithi183 ceph-mon[27058]: Adjusting osd_memory_target on smithi122 to 4288M 2024-04-02T14:50:49.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:48 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:49.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:48 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:49.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:48 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:50:49.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:48 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:50:49.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:48 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:50:49.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:48 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:50:49.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:48 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/1058958620' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "135c10c3-46ab-48b1-967e-fdcfc72eaaa4"}]: dispatch 2024-04-02T14:50:49.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:48 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/1058958620' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "135c10c3-46ab-48b1-967e-fdcfc72eaaa4"}]': finished 2024-04-02T14:50:49.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:48 smithi183 ceph-mon[27058]: osdmap e24: 4 total, 3 up, 4 in 2024-04-02T14:50:49.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:48 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:50:50.339 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:49 smithi122 ceph-mon[24801]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:50:50.339 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:49 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/180626533' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:50:50.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:49 smithi183 ceph-mon[27058]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:50:50.401 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:49 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/180626533' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:50:52.330 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:52 smithi122 ceph-mon[24801]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:50:52.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:52 smithi183 ceph-mon[27058]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:50:54.492 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:54 smithi122 ceph-mon[24801]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:50:54.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:54 smithi183 ceph-mon[27058]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:50:55.814 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:55 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-02T14:50:55.814 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:55 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:50:55.815 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:55 smithi122 ceph-mon[24801]: Deploying daemon osd.3 on smithi122 2024-04-02T14:50:55.815 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:55 smithi122 ceph-mon[24801]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:50:55.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:55 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-02T14:50:55.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:55 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:50:55.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:55 smithi183 ceph-mon[27058]: Deploying daemon osd.3 on smithi122 2024-04-02T14:50:55.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:55 smithi183 ceph-mon[27058]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:50:58.403 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:50:58 smithi122 ceph-mon[24801]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:50:58.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:50:58 smithi183 ceph-mon[27058]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:51:00.284 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:00 smithi122 ceph-mon[24801]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:51:00.285 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:00 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:00.285 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:00 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:51:00.285 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:00 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:00.285 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:00 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:51:00.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:00 smithi183 ceph-mon[27058]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:51:00.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:00 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:00.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:00 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:51:00.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:00 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:00.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:00 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:51:02.504 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:02 smithi122 ceph-mon[24801]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:51:02.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:02 smithi183 ceph-mon[27058]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:51:03.993 INFO:teuthology.orchestra.run.smithi122.stdout:Created osd(s) 3 on host 'smithi122' 2024-04-02T14:51:04.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:04 smithi122 ceph-mon[24801]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:51:04.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:04 smithi122 ceph-mon[24801]: from='osd.3 [v2:172.21.15.122:6826/685608550,v1:172.21.15.122:6827/685608550]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T14:51:04.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:04 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:04.615 DEBUG:teuthology.orchestra.run.smithi122:osd.3> sudo journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.3.service 2024-04-02T14:51:04.618 INFO:tasks.cephadm:Deploying osd.4 on smithi183 with /dev/vg_nvme/lv_4... 2024-04-02T14:51:04.618 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-02T14:51:04.647 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:04 smithi183 ceph-mon[27058]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:51:04.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:04 smithi183 ceph-mon[27058]: from='osd.3 [v2:172.21.15.122:6826/685608550,v1:172.21.15.122:6827/685608550]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T14:51:04.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:04 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:05.540 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:05 smithi122 ceph-mon[24801]: from='osd.3 [v2:172.21.15.122:6826/685608550,v1:172.21.15.122:6827/685608550]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-02T14:51:05.540 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:05 smithi122 ceph-mon[24801]: osdmap e25: 4 total, 3 up, 4 in 2024-04-02T14:51:05.541 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:05 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:51:05.541 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:05 smithi122 ceph-mon[24801]: from='osd.3 [v2:172.21.15.122:6826/685608550,v1:172.21.15.122:6827/685608550]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi122", "root=default"]}]: dispatch 2024-04-02T14:51:05.541 INFO:journalctl@ceph.osd.3.smithi122.stdout:Apr 02 14:51:05 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-3[46177]: 2024-04-02T14:51:05.264+0000 7fb723c83700 -1 osd.3 0 waiting for initial osdmap 2024-04-02T14:51:05.541 INFO:journalctl@ceph.osd.3.smithi122.stdout:Apr 02 14:51:05 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-3[46177]: 2024-04-02T14:51:05.271+0000 7fb71e61a700 -1 osd.3 26 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T14:51:05.575 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:05 smithi183 ceph-mon[27058]: from='osd.3 [v2:172.21.15.122:6826/685608550,v1:172.21.15.122:6827/685608550]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-02T14:51:05.576 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:05 smithi183 ceph-mon[27058]: osdmap e25: 4 total, 3 up, 4 in 2024-04-02T14:51:05.576 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:05 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:51:05.576 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:05 smithi183 ceph-mon[27058]: from='osd.3 [v2:172.21.15.122:6826/685608550,v1:172.21.15.122:6827/685608550]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi122", "root=default"]}]: dispatch 2024-04-02T14:51:06.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:06 smithi122 ceph-mon[24801]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:51:06.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:06 smithi122 ceph-mon[24801]: from='osd.3 [v2:172.21.15.122:6826/685608550,v1:172.21.15.122:6827/685608550]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi122", "root=default"]}]': finished 2024-04-02T14:51:06.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:06 smithi122 ceph-mon[24801]: osdmap e26: 4 total, 3 up, 4 in 2024-04-02T14:51:06.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:06 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:51:06.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:06 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:51:06.619 INFO:teuthology.orchestra.run.smithi183.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-02T14:51:06.619 INFO:teuthology.orchestra.run.smithi183.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-02T14:51:06.619 INFO:teuthology.orchestra.run.smithi183.stderr: stderr: 10+0 records in 2024-04-02T14:51:06.619 INFO:teuthology.orchestra.run.smithi183.stderr:10+0 records out 2024-04-02T14:51:06.619 INFO:teuthology.orchestra.run.smithi183.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0110234 s, 951 MB/s 2024-04-02T14:51:06.619 INFO:teuthology.orchestra.run.smithi183.stderr:--> Zapping successful for: 2024-04-02T14:51:06.620 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:06 smithi183 ceph-mon[27058]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:51:06.620 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:06 smithi183 ceph-mon[27058]: from='osd.3 [v2:172.21.15.122:6826/685608550,v1:172.21.15.122:6827/685608550]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi122", "root=default"]}]': finished 2024-04-02T14:51:06.620 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:06 smithi183 ceph-mon[27058]: osdmap e26: 4 total, 3 up, 4 in 2024-04-02T14:51:06.620 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:06 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:51:06.620 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:06 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:51:07.198 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi183:vg_nvme/lv_4 2024-04-02T14:51:07.508 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:07 smithi122 ceph-mon[24801]: purged_snaps scrub starts 2024-04-02T14:51:07.508 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:07 smithi122 ceph-mon[24801]: purged_snaps scrub ok 2024-04-02T14:51:07.508 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:07 smithi122 ceph-mon[24801]: osd.3 [v2:172.21.15.122:6826/685608550,v1:172.21.15.122:6827/685608550] boot 2024-04-02T14:51:07.508 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:07 smithi122 ceph-mon[24801]: osdmap e27: 4 total, 4 up, 4 in 2024-04-02T14:51:07.508 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:07 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:51:07.508 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:07 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:07.554 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:07 smithi183 ceph-mon[27058]: purged_snaps scrub starts 2024-04-02T14:51:07.554 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:07 smithi183 ceph-mon[27058]: purged_snaps scrub ok 2024-04-02T14:51:07.554 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:07 smithi183 ceph-mon[27058]: osd.3 [v2:172.21.15.122:6826/685608550,v1:172.21.15.122:6827/685608550] boot 2024-04-02T14:51:07.554 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:07 smithi183 ceph-mon[27058]: osdmap e27: 4 total, 4 up, 4 in 2024-04-02T14:51:07.554 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:07 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:51:07.554 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:07 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:08.551 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:08 smithi122 ceph-mon[24801]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:51:08.551 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:08 smithi122 ceph-mon[24801]: osdmap e28: 4 total, 4 up, 4 in 2024-04-02T14:51:08.559 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:08 smithi183 ceph-mon[27058]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T14:51:08.559 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:08 smithi183 ceph-mon[27058]: osdmap e28: 4 total, 4 up, 4 in 2024-04-02T14:51:09.574 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:09 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:51:09.575 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:09 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:51:09.575 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:09 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:09.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:09 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:51:09.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:09 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:51:09.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:09 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:10.505 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:10 smithi183 ceph-mon[27058]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:10.505 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:10 smithi183 ceph-mon[27058]: from='client.24137 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi183:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:51:10.505 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:10 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:10.506 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:10 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:51:10.506 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:10 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:51:10.506 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:10 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:51:10.506 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:10 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:51:10.506 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:10 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:10.506 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:10 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:10.506 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:10 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:51:10.506 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:10 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:10.506 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:10 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:51:10.506 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:10 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:10.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:10 smithi122 ceph-mon[24801]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:10.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:10 smithi122 ceph-mon[24801]: from='client.24137 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi183:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:51:10.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:10 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:10.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:10 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:51:10.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:10 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:51:10.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:10 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:51:10.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:10 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:51:10.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:10 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:10.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:10 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:10.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:10 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:51:10.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:10 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:10.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:10 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:51:10.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:10 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:11.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:11 smithi122 ceph-mon[24801]: Detected new or changed devices on smithi122 2024-04-02T14:51:11.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:11 smithi122 ceph-mon[24801]: Adjusting osd_memory_target on smithi122 to 3216M 2024-04-02T14:51:11.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:11 smithi183 ceph-mon[27058]: Detected new or changed devices on smithi122 2024-04-02T14:51:11.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:11 smithi183 ceph-mon[27058]: Adjusting osd_memory_target on smithi122 to 3216M 2024-04-02T14:51:12.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:12 smithi122 ceph-mon[24801]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:12.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:12 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/3963428677' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d3a1fe32-a760-498b-94a3-a37e342ec4db"}]: dispatch 2024-04-02T14:51:12.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:12 smithi122 ceph-mon[24801]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d3a1fe32-a760-498b-94a3-a37e342ec4db"}]: dispatch 2024-04-02T14:51:12.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:12 smithi122 ceph-mon[24801]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d3a1fe32-a760-498b-94a3-a37e342ec4db"}]': finished 2024-04-02T14:51:12.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:12 smithi122 ceph-mon[24801]: osdmap e29: 5 total, 4 up, 5 in 2024-04-02T14:51:12.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:12 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:51:12.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:12 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/1363617047' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:51:12.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:12 smithi183 ceph-mon[27058]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:12.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:12 smithi183 ceph-mon[27058]: from='client.? 172.21.15.183:0/3963428677' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d3a1fe32-a760-498b-94a3-a37e342ec4db"}]: dispatch 2024-04-02T14:51:12.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:12 smithi183 ceph-mon[27058]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d3a1fe32-a760-498b-94a3-a37e342ec4db"}]: dispatch 2024-04-02T14:51:12.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:12 smithi183 ceph-mon[27058]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d3a1fe32-a760-498b-94a3-a37e342ec4db"}]': finished 2024-04-02T14:51:12.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:12 smithi183 ceph-mon[27058]: osdmap e29: 5 total, 4 up, 5 in 2024-04-02T14:51:12.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:12 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:51:12.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:12 smithi183 ceph-mon[27058]: from='client.? 172.21.15.183:0/1363617047' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:51:14.579 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:14 smithi183 ceph-mon[27058]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:14.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:14 smithi122 ceph-mon[24801]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:15.618 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:15 smithi183 ceph-mon[27058]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:15.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:15 smithi122 ceph-mon[24801]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:18.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:18 smithi122 ceph-mon[24801]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:18.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:18 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-02T14:51:18.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:18 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:18.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:18 smithi183 ceph-mon[27058]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:18.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:18 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-02T14:51:18.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:18 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:19.486 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:19 smithi183 ceph-mon[27058]: Deploying daemon osd.4 on smithi183 2024-04-02T14:51:19.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:19 smithi122 ceph-mon[24801]: Deploying daemon osd.4 on smithi183 2024-04-02T14:51:20.281 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:20 smithi183 ceph-mon[27058]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:20.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:20 smithi122 ceph-mon[24801]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:22.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:22 smithi183 ceph-mon[27058]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:22.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:22 smithi122 ceph-mon[24801]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:24.063 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:23 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:24.063 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:23 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:51:24.063 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:23 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:24.063 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:23 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:51:24.063 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:23 smithi183 ceph-mon[27058]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:24.089 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:23 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:24.089 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:23 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:51:24.089 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:23 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:24.089 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:23 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:51:24.089 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:23 smithi122 ceph-mon[24801]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:25.444 INFO:teuthology.orchestra.run.smithi183.stdout:Created osd(s) 4 on host 'smithi183' 2024-04-02T14:51:25.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:25 smithi183 ceph-mon[27058]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:25.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:25 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:25.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:25 smithi122 ceph-mon[24801]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:25.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:25 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:26.423 DEBUG:teuthology.orchestra.run.smithi183:osd.4> sudo journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.4.service 2024-04-02T14:51:26.425 INFO:tasks.cephadm:Deploying osd.5 on smithi183 with /dev/vg_nvme/lv_3... 2024-04-02T14:51:26.426 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-02T14:51:26.778 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:26 smithi183 ceph-mon[27058]: from='osd.4 [v2:172.21.15.183:6800/3537677704,v1:172.21.15.183:6801/3537677704]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T14:51:26.779 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:26 smithi183 ceph-mon[27058]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T14:51:26.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:26 smithi122 ceph-mon[24801]: from='osd.4 [v2:172.21.15.183:6800/3537677704,v1:172.21.15.183:6801/3537677704]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T14:51:26.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:26 smithi122 ceph-mon[24801]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T14:51:27.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:27 smithi183 ceph-mon[27058]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-02T14:51:27.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:27 smithi183 ceph-mon[27058]: osdmap e30: 5 total, 4 up, 5 in 2024-04-02T14:51:27.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:27 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:51:27.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:27 smithi183 ceph-mon[27058]: from='osd.4 [v2:172.21.15.183:6800/3537677704,v1:172.21.15.183:6801/3537677704]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]: dispatch 2024-04-02T14:51:27.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:27 smithi183 ceph-mon[27058]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]: dispatch 2024-04-02T14:51:27.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:27 smithi183 ceph-mon[27058]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:27.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:27 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:27.651 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:51:27 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4[31295]: 2024-04-02T14:51:27.559+0000 7f8376350700 -1 osd.4 0 waiting for initial osdmap 2024-04-02T14:51:27.651 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:51:27 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4[31295]: 2024-04-02T14:51:27.563+0000 7f8371ce9700 -1 osd.4 31 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T14:51:27.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:27 smithi122 ceph-mon[24801]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-02T14:51:27.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:27 smithi122 ceph-mon[24801]: osdmap e30: 5 total, 4 up, 5 in 2024-04-02T14:51:27.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:27 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:51:27.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:27 smithi122 ceph-mon[24801]: from='osd.4 [v2:172.21.15.183:6800/3537677704,v1:172.21.15.183:6801/3537677704]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]: dispatch 2024-04-02T14:51:27.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:27 smithi122 ceph-mon[24801]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]: dispatch 2024-04-02T14:51:27.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:27 smithi122 ceph-mon[24801]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:27.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:27 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:28.755 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:28 smithi183 ceph-mon[27058]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]': finished 2024-04-02T14:51:28.755 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:28 smithi183 ceph-mon[27058]: osdmap e31: 5 total, 4 up, 5 in 2024-04-02T14:51:28.755 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:28 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:51:28.755 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:28 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:51:28.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:28 smithi122 ceph-mon[24801]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]': finished 2024-04-02T14:51:28.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:28 smithi122 ceph-mon[24801]: osdmap e31: 5 total, 4 up, 5 in 2024-04-02T14:51:28.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:28 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:51:28.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:28 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:51:29.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:29 smithi183 ceph-mon[27058]: purged_snaps scrub starts 2024-04-02T14:51:29.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:29 smithi183 ceph-mon[27058]: purged_snaps scrub ok 2024-04-02T14:51:29.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:29 smithi183 ceph-mon[27058]: osd.4 [v2:172.21.15.183:6800/3537677704,v1:172.21.15.183:6801/3537677704] boot 2024-04-02T14:51:29.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:29 smithi183 ceph-mon[27058]: osdmap e32: 5 total, 5 up, 5 in 2024-04-02T14:51:29.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:51:29.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:29 smithi183 ceph-mon[27058]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:29.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/trash_purge_schedule"}]: dispatch 2024-04-02T14:51:29.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:51:29.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:29 smithi183 ceph-mon[27058]: osdmap e33: 5 total, 5 up, 5 in 2024-04-02T14:51:29.715 INFO:teuthology.orchestra.run.smithi183.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-02T14:51:29.715 INFO:teuthology.orchestra.run.smithi183.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-02T14:51:29.715 INFO:teuthology.orchestra.run.smithi183.stderr: stderr: 10+0 records in 2024-04-02T14:51:29.715 INFO:teuthology.orchestra.run.smithi183.stderr:10+0 records out 2024-04-02T14:51:29.715 INFO:teuthology.orchestra.run.smithi183.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0162634 s, 645 MB/s 2024-04-02T14:51:29.715 INFO:teuthology.orchestra.run.smithi183.stderr:--> Zapping successful for: 2024-04-02T14:51:29.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:29 smithi122 ceph-mon[24801]: purged_snaps scrub starts 2024-04-02T14:51:29.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:29 smithi122 ceph-mon[24801]: purged_snaps scrub ok 2024-04-02T14:51:29.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:29 smithi122 ceph-mon[24801]: osd.4 [v2:172.21.15.183:6800/3537677704,v1:172.21.15.183:6801/3537677704] boot 2024-04-02T14:51:29.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:29 smithi122 ceph-mon[24801]: osdmap e32: 5 total, 5 up, 5 in 2024-04-02T14:51:29.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:51:29.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:29 smithi122 ceph-mon[24801]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T14:51:29.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/trash_purge_schedule"}]: dispatch 2024-04-02T14:51:29.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:51:29.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:29 smithi122 ceph-mon[24801]: osdmap e33: 5 total, 5 up, 5 in 2024-04-02T14:51:30.615 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi183:vg_nvme/lv_3 2024-04-02T14:51:31.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:31 smithi122 ceph-mon[24801]: osdmap e34: 5 total, 5 up, 5 in 2024-04-02T14:51:31.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:31 smithi122 ceph-mon[24801]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:51:31.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:31 smithi183 ceph-mon[27058]: osdmap e34: 5 total, 5 up, 5 in 2024-04-02T14:51:31.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:31 smithi183 ceph-mon[27058]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:51:32.727 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:32 smithi183 ceph-mon[27058]: osdmap e35: 5 total, 5 up, 5 in 2024-04-02T14:51:32.727 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:32 smithi183 ceph-mon[27058]: Detected new or changed devices on smithi183 2024-04-02T14:51:32.727 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:32 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:32.727 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:32 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:51:32.727 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:32 smithi183 ceph-mon[27058]: Adjusting osd_memory_target on smithi183 to 15936M 2024-04-02T14:51:32.727 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:32 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:32.727 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:32 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:32.727 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:32 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:51:32.728 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:32 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:32.728 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:32 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:51:32.728 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:32 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:32.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:32 smithi122 ceph-mon[24801]: osdmap e35: 5 total, 5 up, 5 in 2024-04-02T14:51:32.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:32 smithi122 ceph-mon[24801]: Detected new or changed devices on smithi183 2024-04-02T14:51:32.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:32.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:51:32.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:32 smithi122 ceph-mon[24801]: Adjusting osd_memory_target on smithi183 to 15936M 2024-04-02T14:51:32.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:32.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:32.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:51:32.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:32.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:51:32.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:33.755 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:33 smithi183 ceph-mon[27058]: osdmap e36: 5 total, 5 up, 5 in 2024-04-02T14:51:33.755 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:33 smithi183 ceph-mon[27058]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-02T14:51:33.756 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:33 smithi183 ceph-mon[27058]: from='client.24155 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi183:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:51:33.756 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:33 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:51:33.756 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:33 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:51:33.756 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:33 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:33.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:33 smithi122 ceph-mon[24801]: osdmap e36: 5 total, 5 up, 5 in 2024-04-02T14:51:33.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:33 smithi122 ceph-mon[24801]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-02T14:51:33.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:33 smithi122 ceph-mon[24801]: from='client.24155 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi183:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:51:33.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:33 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:51:33.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:33 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:51:33.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:33 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:35.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:35 smithi122 ceph-mon[24801]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-02T14:51:35.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:35 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/1548536989' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9393d535-0a63-462f-9db6-a461b1c21dbe"}]: dispatch 2024-04-02T14:51:35.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:35 smithi122 ceph-mon[24801]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9393d535-0a63-462f-9db6-a461b1c21dbe"}]: dispatch 2024-04-02T14:51:35.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:35 smithi122 ceph-mon[24801]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9393d535-0a63-462f-9db6-a461b1c21dbe"}]': finished 2024-04-02T14:51:35.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:35 smithi122 ceph-mon[24801]: osdmap e37: 6 total, 5 up, 6 in 2024-04-02T14:51:35.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:35 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:51:35.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:35 smithi183 ceph-mon[27058]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-02T14:51:35.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:35 smithi183 ceph-mon[27058]: from='client.? 172.21.15.183:0/1548536989' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9393d535-0a63-462f-9db6-a461b1c21dbe"}]: dispatch 2024-04-02T14:51:35.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:35 smithi183 ceph-mon[27058]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9393d535-0a63-462f-9db6-a461b1c21dbe"}]: dispatch 2024-04-02T14:51:35.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:35 smithi183 ceph-mon[27058]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9393d535-0a63-462f-9db6-a461b1c21dbe"}]': finished 2024-04-02T14:51:35.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:35 smithi183 ceph-mon[27058]: osdmap e37: 6 total, 5 up, 6 in 2024-04-02T14:51:35.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:35 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:51:36.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:36 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/253824364' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:51:36.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:36 smithi183 ceph-mon[27058]: from='client.? 172.21.15.183:0/253824364' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:51:37.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:37 smithi122 ceph-mon[24801]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T14:51:37.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:37 smithi183 ceph-mon[27058]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T14:51:40.282 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:40 smithi183 ceph-mon[27058]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-04-02T14:51:40.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:40 smithi122 ceph-mon[24801]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-04-02T14:51:42.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:42 smithi122 ceph-mon[24801]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-04-02T14:51:42.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:42 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-02T14:51:42.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:42 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:42.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:42 smithi183 ceph-mon[27058]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-04-02T14:51:42.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:42 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-02T14:51:42.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:42 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:43.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:43 smithi122 ceph-mon[24801]: Deploying daemon osd.5 on smithi183 2024-04-02T14:51:43.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:43 smithi183 ceph-mon[27058]: Deploying daemon osd.5 on smithi183 2024-04-02T14:51:44.547 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:44 smithi183 ceph-mon[27058]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:51:44.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:44 smithi122 ceph-mon[24801]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:51:45.765 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:45 smithi183 ceph-mon[27058]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:51:45.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:45 smithi122 ceph-mon[24801]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:51:47.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:47 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:47.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:47 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:51:47.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:47 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:47.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:47 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:51:47.640 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:47 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:47.640 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:47 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:51:47.640 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:47 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:47.640 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:47 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:51:48.546 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:48 smithi183 ceph-mon[27058]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:51:48.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:48 smithi122 ceph-mon[24801]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:51:49.351 INFO:teuthology.orchestra.run.smithi183.stdout:Created osd(s) 5 on host 'smithi183' 2024-04-02T14:51:50.100 DEBUG:teuthology.orchestra.run.smithi183:osd.5> sudo journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.5.service 2024-04-02T14:51:50.102 INFO:tasks.cephadm:Deploying osd.6 on smithi183 with /dev/vg_nvme/lv_2... 2024-04-02T14:51:50.102 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-02T14:51:50.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:50 smithi183 ceph-mon[27058]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:51:50.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:50 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:50.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:50 smithi183 ceph-mon[27058]: from='osd.5 [v2:172.21.15.183:6808/273662903,v1:172.21.15.183:6809/273662903]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T14:51:50.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:50 smithi183 ceph-mon[27058]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T14:51:50.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:50 smithi122 ceph-mon[24801]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:51:50.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:50 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:50.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:50 smithi122 ceph-mon[24801]: from='osd.5 [v2:172.21.15.183:6808/273662903,v1:172.21.15.183:6809/273662903]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T14:51:50.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:50 smithi122 ceph-mon[24801]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T14:51:51.577 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:51 smithi183 ceph-mon[27058]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-02T14:51:51.577 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:51 smithi183 ceph-mon[27058]: osdmap e38: 6 total, 5 up, 6 in 2024-04-02T14:51:51.577 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:51 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:51:51.577 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:51 smithi183 ceph-mon[27058]: from='osd.5 [v2:172.21.15.183:6808/273662903,v1:172.21.15.183:6809/273662903]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]: dispatch 2024-04-02T14:51:51.577 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:51 smithi183 ceph-mon[27058]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]: dispatch 2024-04-02T14:51:51.577 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:51 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:51.577 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:51:51 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:51:51.379+0000 7f892acf7700 -1 osd.5 0 waiting for initial osdmap 2024-04-02T14:51:51.577 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:51:51 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:51:51.385+0000 7f8924e8d700 -1 osd.5 39 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T14:51:51.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:51 smithi122 ceph-mon[24801]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-02T14:51:51.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:51 smithi122 ceph-mon[24801]: osdmap e38: 6 total, 5 up, 6 in 2024-04-02T14:51:51.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:51 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:51:51.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:51 smithi122 ceph-mon[24801]: from='osd.5 [v2:172.21.15.183:6808/273662903,v1:172.21.15.183:6809/273662903]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]: dispatch 2024-04-02T14:51:51.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:51 smithi122 ceph-mon[24801]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]: dispatch 2024-04-02T14:51:51.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:51 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:52.536 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:52 smithi183 ceph-mon[27058]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:51:52.536 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:52 smithi183 ceph-mon[27058]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]': finished 2024-04-02T14:51:52.536 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:52 smithi183 ceph-mon[27058]: osdmap e39: 6 total, 5 up, 6 in 2024-04-02T14:51:52.536 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:52 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:51:52.536 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:52 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:51:52.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:52 smithi122 ceph-mon[24801]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T14:51:52.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:52 smithi122 ceph-mon[24801]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]': finished 2024-04-02T14:51:52.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:52 smithi122 ceph-mon[24801]: osdmap e39: 6 total, 5 up, 6 in 2024-04-02T14:51:52.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:52 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:51:52.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:52 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:51:53.367 INFO:teuthology.orchestra.run.smithi183.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-02T14:51:53.367 INFO:teuthology.orchestra.run.smithi183.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-02T14:51:53.367 INFO:teuthology.orchestra.run.smithi183.stderr: stderr: 10+0 records in 2024-04-02T14:51:53.367 INFO:teuthology.orchestra.run.smithi183.stderr:10+0 records out 2024-04-02T14:51:53.367 INFO:teuthology.orchestra.run.smithi183.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0171192 s, 613 MB/s 2024-04-02T14:51:53.367 INFO:teuthology.orchestra.run.smithi183.stderr:--> Zapping successful for: 2024-04-02T14:51:53.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:53 smithi183 ceph-mon[27058]: purged_snaps scrub starts 2024-04-02T14:51:53.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:53 smithi183 ceph-mon[27058]: purged_snaps scrub ok 2024-04-02T14:51:53.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:53 smithi183 ceph-mon[27058]: osd.5 [v2:172.21.15.183:6808/273662903,v1:172.21.15.183:6809/273662903] boot 2024-04-02T14:51:53.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:53 smithi183 ceph-mon[27058]: osdmap e40: 6 total, 6 up, 6 in 2024-04-02T14:51:53.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:53 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:51:53.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:53 smithi183 ceph-mon[27058]: osdmap e41: 6 total, 6 up, 6 in 2024-04-02T14:51:53.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:53 smithi122 ceph-mon[24801]: purged_snaps scrub starts 2024-04-02T14:51:53.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:53 smithi122 ceph-mon[24801]: purged_snaps scrub ok 2024-04-02T14:51:53.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:53 smithi122 ceph-mon[24801]: osd.5 [v2:172.21.15.183:6808/273662903,v1:172.21.15.183:6809/273662903] boot 2024-04-02T14:51:53.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:53 smithi122 ceph-mon[24801]: osdmap e40: 6 total, 6 up, 6 in 2024-04-02T14:51:53.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:53 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:51:53.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:53 smithi122 ceph-mon[24801]: osdmap e41: 6 total, 6 up, 6 in 2024-04-02T14:51:54.074 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi183:vg_nvme/lv_2 2024-04-02T14:51:54.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:54 smithi183 ceph-mon[27058]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:51:54.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:54 smithi183 ceph-mon[27058]: osdmap e42: 6 total, 6 up, 6 in 2024-04-02T14:51:54.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:54 smithi122 ceph-mon[24801]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:51:54.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:54 smithi122 ceph-mon[24801]: osdmap e42: 6 total, 6 up, 6 in 2024-04-02T14:51:55.702 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:55 smithi183 ceph-mon[27058]: osdmap e43: 6 total, 6 up, 6 in 2024-04-02T14:51:55.702 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:55 smithi183 ceph-mon[27058]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:51:55.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:55 smithi122 ceph-mon[24801]: osdmap e43: 6 total, 6 up, 6 in 2024-04-02T14:51:55.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:55 smithi122 ceph-mon[24801]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:51:56.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:56 smithi122 ceph-mon[24801]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-02T14:51:56.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:56 smithi122 ceph-mon[24801]: osdmap e44: 6 total, 6 up, 6 in 2024-04-02T14:51:56.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:56 smithi122 ceph-mon[24801]: Detected new or changed devices on smithi183 2024-04-02T14:51:56.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:56 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:56.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:56 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:51:56.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:56 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:51:56.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:56 smithi122 ceph-mon[24801]: Adjusting osd_memory_target on smithi183 to 7968M 2024-04-02T14:51:56.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:56 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:56.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:56 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:56.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:56 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:51:56.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:56 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:56.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:56 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:51:56.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:56 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:56.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:56 smithi183 ceph-mon[27058]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-02T14:51:56.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:56 smithi183 ceph-mon[27058]: osdmap e44: 6 total, 6 up, 6 in 2024-04-02T14:51:56.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:56 smithi183 ceph-mon[27058]: Detected new or changed devices on smithi183 2024-04-02T14:51:56.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:56 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:56.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:56 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:51:56.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:56 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:51:56.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:56 smithi183 ceph-mon[27058]: Adjusting osd_memory_target on smithi183 to 7968M 2024-04-02T14:51:56.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:56 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:56.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:56 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:56.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:56 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:51:56.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:56 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:56.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:56 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:51:56.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:56 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:51:57.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:57 smithi122 ceph-mon[24801]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi183:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:51:57.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:57 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:51:57.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:57 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:51:57.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:57 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:57.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:57 smithi122 ceph-mon[24801]: pgmap v136: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:51:57.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:57 smithi183 ceph-mon[27058]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi183:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:51:57.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:57 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:51:57.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:57 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:51:57.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:57 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:51:57.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:57 smithi183 ceph-mon[27058]: pgmap v136: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:51:59.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:59 smithi183 ceph-mon[27058]: from='client.? 172.21.15.183:0/2737091074' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4529b54e-730d-4036-b885-da2b4770c7d5"}]: dispatch 2024-04-02T14:51:59.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:59 smithi183 ceph-mon[27058]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4529b54e-730d-4036-b885-da2b4770c7d5"}]: dispatch 2024-04-02T14:51:59.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:59 smithi183 ceph-mon[27058]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4529b54e-730d-4036-b885-da2b4770c7d5"}]': finished 2024-04-02T14:51:59.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:59 smithi183 ceph-mon[27058]: osdmap e45: 7 total, 6 up, 7 in 2024-04-02T14:51:59.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:51:59 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T14:51:59.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:59 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/2737091074' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4529b54e-730d-4036-b885-da2b4770c7d5"}]: dispatch 2024-04-02T14:51:59.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:59 smithi122 ceph-mon[24801]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4529b54e-730d-4036-b885-da2b4770c7d5"}]: dispatch 2024-04-02T14:51:59.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:59 smithi122 ceph-mon[24801]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4529b54e-730d-4036-b885-da2b4770c7d5"}]': finished 2024-04-02T14:51:59.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:59 smithi122 ceph-mon[24801]: osdmap e45: 7 total, 6 up, 7 in 2024-04-02T14:51:59.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:51:59 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T14:52:00.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:00 smithi122 ceph-mon[24801]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-04-02T14:52:00.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:00 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:00.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:00 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/2468347047' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:52:00.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:00 smithi122 ceph-mon[24801]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-02T14:52:00.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:00 smithi122 ceph-mon[24801]: Cluster is now healthy 2024-04-02T14:52:00.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:00 smithi183 ceph-mon[27058]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-04-02T14:52:00.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:00 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:00.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:00 smithi183 ceph-mon[27058]: from='client.? 172.21.15.183:0/2468347047' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:52:00.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:00 smithi183 ceph-mon[27058]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-02T14:52:00.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:00 smithi183 ceph-mon[27058]: Cluster is now healthy 2024-04-02T14:52:02.488 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:02 smithi183 ceph-mon[27058]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-04-02T14:52:02.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:02 smithi122 ceph-mon[24801]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-04-02T14:52:04.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:04 smithi183 ceph-mon[27058]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-02T14:52:04.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:04 smithi122 ceph-mon[24801]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-02T14:52:05.799 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:05 smithi183 ceph-mon[27058]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-04-02T14:52:05.799 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:05 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-02T14:52:05.799 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:05 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:52:05.799 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:05 smithi183 ceph-mon[27058]: Deploying daemon osd.6 on smithi183 2024-04-02T14:52:05.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:05 smithi122 ceph-mon[24801]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-04-02T14:52:05.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:05 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-02T14:52:05.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:05 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:52:05.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:05 smithi122 ceph-mon[24801]: Deploying daemon osd.6 on smithi183 2024-04-02T14:52:08.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:08 smithi183 ceph-mon[27058]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-02T14:52:08.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:08 smithi122 ceph-mon[24801]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-02T14:52:10.372 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:10 smithi183 ceph-mon[27058]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:52:10.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:10 smithi122 ceph-mon[24801]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:52:11.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:11 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:11.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:11 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:52:11.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:11 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:52:11.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:11 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:52:11.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:11 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:11.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:11 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:52:11.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:11 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:52:11.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:11 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:52:12.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:12 smithi183 ceph-mon[27058]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:52:12.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:12 smithi122 ceph-mon[24801]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:52:13.406 INFO:teuthology.orchestra.run.smithi183.stdout:Created osd(s) 6 on host 'smithi183' 2024-04-02T14:52:14.025 DEBUG:teuthology.orchestra.run.smithi183:osd.6> sudo journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.6.service 2024-04-02T14:52:14.027 INFO:tasks.cephadm:Deploying osd.7 on smithi183 with /dev/vg_nvme/lv_1... 2024-04-02T14:52:14.028 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-02T14:52:14.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:14 smithi183 ceph-mon[27058]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:52:14.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:14 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:14.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:14 smithi183 ceph-mon[27058]: from='osd.6 [v2:172.21.15.183:6816/3624098587,v1:172.21.15.183:6817/3624098587]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T14:52:14.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:14 smithi183 ceph-mon[27058]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T14:52:14.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:14 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:14.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:14 smithi122 ceph-mon[24801]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:52:14.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:14 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:14.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:14 smithi122 ceph-mon[24801]: from='osd.6 [v2:172.21.15.183:6816/3624098587,v1:172.21.15.183:6817/3624098587]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T14:52:14.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:14 smithi122 ceph-mon[24801]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T14:52:14.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:14 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:15.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:15 smithi183 ceph-mon[27058]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-02T14:52:15.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:15 smithi183 ceph-mon[27058]: osdmap e46: 7 total, 6 up, 7 in 2024-04-02T14:52:15.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:15 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T14:52:15.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:15 smithi183 ceph-mon[27058]: from='osd.6 [v2:172.21.15.183:6816/3624098587,v1:172.21.15.183:6817/3624098587]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]: dispatch 2024-04-02T14:52:15.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:15 smithi183 ceph-mon[27058]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]: dispatch 2024-04-02T14:52:15.650 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:52:15 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:52:15.428+0000 7ff67f68a700 -1 osd.6 0 waiting for initial osdmap 2024-04-02T14:52:15.650 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:52:15 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:52:15.434+0000 7ff67a021700 -1 osd.6 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T14:52:15.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:15 smithi122 ceph-mon[24801]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-02T14:52:15.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:15 smithi122 ceph-mon[24801]: osdmap e46: 7 total, 6 up, 7 in 2024-04-02T14:52:15.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:15 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T14:52:15.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:15 smithi122 ceph-mon[24801]: from='osd.6 [v2:172.21.15.183:6816/3624098587,v1:172.21.15.183:6817/3624098587]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]: dispatch 2024-04-02T14:52:15.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:15 smithi122 ceph-mon[24801]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]: dispatch 2024-04-02T14:52:16.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:16 smithi183 ceph-mon[27058]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:52:16.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:16 smithi183 ceph-mon[27058]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]': finished 2024-04-02T14:52:16.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:16 smithi183 ceph-mon[27058]: osdmap e47: 7 total, 6 up, 7 in 2024-04-02T14:52:16.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:16 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T14:52:16.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:16 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T14:52:16.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:16 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:16.781 INFO:teuthology.orchestra.run.smithi183.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-02T14:52:16.781 INFO:teuthology.orchestra.run.smithi183.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-02T14:52:16.781 INFO:teuthology.orchestra.run.smithi183.stderr: stderr: 10+0 records in 2024-04-02T14:52:16.781 INFO:teuthology.orchestra.run.smithi183.stderr:10+0 records out 2024-04-02T14:52:16.781 INFO:teuthology.orchestra.run.smithi183.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0124179 s, 844 MB/s 2024-04-02T14:52:16.782 INFO:teuthology.orchestra.run.smithi183.stderr:--> Zapping successful for: 2024-04-02T14:52:16.838 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:16 smithi122 ceph-mon[24801]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:52:16.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:16 smithi122 ceph-mon[24801]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]': finished 2024-04-02T14:52:16.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:16 smithi122 ceph-mon[24801]: osdmap e47: 7 total, 6 up, 7 in 2024-04-02T14:52:16.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:16 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T14:52:16.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:16 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T14:52:16.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:16 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:17.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:17 smithi122 ceph-mon[24801]: purged_snaps scrub starts 2024-04-02T14:52:17.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:17 smithi122 ceph-mon[24801]: purged_snaps scrub ok 2024-04-02T14:52:17.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:17 smithi122 ceph-mon[24801]: osd.6 [v2:172.21.15.183:6816/3624098587,v1:172.21.15.183:6817/3624098587] boot 2024-04-02T14:52:17.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:17 smithi122 ceph-mon[24801]: osdmap e48: 7 total, 7 up, 7 in 2024-04-02T14:52:17.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:17 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T14:52:17.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:17 smithi122 ceph-mon[24801]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:52:17.800 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:17 smithi183 ceph-mon[27058]: purged_snaps scrub starts 2024-04-02T14:52:17.800 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:17 smithi183 ceph-mon[27058]: purged_snaps scrub ok 2024-04-02T14:52:17.800 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:17 smithi183 ceph-mon[27058]: osd.6 [v2:172.21.15.183:6816/3624098587,v1:172.21.15.183:6817/3624098587] boot 2024-04-02T14:52:17.800 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:17 smithi183 ceph-mon[27058]: osdmap e48: 7 total, 7 up, 7 in 2024-04-02T14:52:17.800 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:17 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T14:52:17.800 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:17 smithi183 ceph-mon[27058]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T14:52:17.824 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi183:vg_nvme/lv_1 2024-04-02T14:52:18.497 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:18 smithi183 ceph-mon[27058]: osdmap e49: 7 total, 7 up, 7 in 2024-04-02T14:52:18.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:18 smithi122 ceph-mon[24801]: osdmap e49: 7 total, 7 up, 7 in 2024-04-02T14:52:19.718 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:19 smithi183 ceph-mon[27058]: osdmap e50: 7 total, 7 up, 7 in 2024-04-02T14:52:19.718 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:19 smithi183 ceph-mon[27058]: pgmap v153: 1 pgs: 1 active+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-04-02T14:52:19.842 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:19 smithi122 ceph-mon[24801]: osdmap e50: 7 total, 7 up, 7 in 2024-04-02T14:52:19.842 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:19 smithi122 ceph-mon[24801]: pgmap v153: 1 pgs: 1 active+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-04-02T14:52:20.677 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:20 smithi183 ceph-mon[27058]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-04-02T14:52:20.677 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:20 smithi183 ceph-mon[27058]: osdmap e51: 7 total, 7 up, 7 in 2024-04-02T14:52:20.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:20 smithi122 ceph-mon[24801]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-04-02T14:52:20.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:20 smithi122 ceph-mon[24801]: osdmap e51: 7 total, 7 up, 7 in 2024-04-02T14:52:21.753 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:21 smithi183 ceph-mon[27058]: osdmap e52: 7 total, 7 up, 7 in 2024-04-02T14:52:21.753 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:21 smithi183 ceph-mon[27058]: Detected new or changed devices on smithi183 2024-04-02T14:52:21.753 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:21 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:21.753 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:21 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:52:21.753 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:21 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:52:21.753 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:21 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:52:21.753 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:21 smithi183 ceph-mon[27058]: Adjusting osd_memory_target on smithi183 to 5312M 2024-04-02T14:52:21.753 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:21 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:21.753 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:21 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:21.753 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:21 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:52:21.753 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:21 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:52:21.753 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:21 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:52:21.753 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:21 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:21.754 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:21 smithi183 ceph-mon[27058]: from='client.24191 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi183:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:52:21.754 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:21 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:52:21.754 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:21 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:52:21.754 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:21 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:52:21.754 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:21 smithi183 ceph-mon[27058]: pgmap v156: 1 pgs: 1 active+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-04-02T14:52:21.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:21 smithi122 ceph-mon[24801]: osdmap e52: 7 total, 7 up, 7 in 2024-04-02T14:52:21.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:21 smithi122 ceph-mon[24801]: Detected new or changed devices on smithi183 2024-04-02T14:52:21.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:21 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:21.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:21 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:52:21.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:21 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:52:21.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:21 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:52:21.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:21 smithi122 ceph-mon[24801]: Adjusting osd_memory_target on smithi183 to 5312M 2024-04-02T14:52:21.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:21 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:21.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:21 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:21.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:21 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:52:21.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:21 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:52:21.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:21 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:52:21.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:21 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:21.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:21 smithi122 ceph-mon[24801]: from='client.24191 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi183:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:52:21.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:21 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T14:52:21.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:21 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T14:52:21.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:21 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:52:21.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:21 smithi122 ceph-mon[24801]: pgmap v156: 1 pgs: 1 active+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-04-02T14:52:23.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:23 smithi122 ceph-mon[24801]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-04-02T14:52:23.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:23 smithi122 ceph-mon[24801]: Cluster is now healthy 2024-04-02T14:52:23.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:23 smithi183 ceph-mon[27058]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-04-02T14:52:23.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:23 smithi183 ceph-mon[27058]: Cluster is now healthy 2024-04-02T14:52:24.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:24 smithi122 ceph-mon[24801]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-04-02T14:52:24.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:24 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/810470556' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cabca570-66f6-4c9d-b8e2-5cc250be64e4"}]: dispatch 2024-04-02T14:52:24.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:24 smithi122 ceph-mon[24801]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cabca570-66f6-4c9d-b8e2-5cc250be64e4"}]: dispatch 2024-04-02T14:52:24.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:24 smithi122 ceph-mon[24801]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cabca570-66f6-4c9d-b8e2-5cc250be64e4"}]': finished 2024-04-02T14:52:24.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:24 smithi122 ceph-mon[24801]: osdmap e53: 8 total, 7 up, 8 in 2024-04-02T14:52:24.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:24 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T14:52:24.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:24 smithi122 ceph-mon[24801]: from='client.? 172.21.15.183:0/3377154409' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:52:24.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:24 smithi183 ceph-mon[27058]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-04-02T14:52:24.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:24 smithi183 ceph-mon[27058]: from='client.? 172.21.15.183:0/810470556' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cabca570-66f6-4c9d-b8e2-5cc250be64e4"}]: dispatch 2024-04-02T14:52:24.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:24 smithi183 ceph-mon[27058]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cabca570-66f6-4c9d-b8e2-5cc250be64e4"}]: dispatch 2024-04-02T14:52:24.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:24 smithi183 ceph-mon[27058]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cabca570-66f6-4c9d-b8e2-5cc250be64e4"}]': finished 2024-04-02T14:52:24.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:24 smithi183 ceph-mon[27058]: osdmap e53: 8 total, 7 up, 8 in 2024-04-02T14:52:24.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:24 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T14:52:24.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:24 smithi183 ceph-mon[27058]: from='client.? 172.21.15.183:0/3377154409' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T14:52:25.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:25 smithi122 ceph-mon[24801]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T14:52:25.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:25 smithi183 ceph-mon[27058]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T14:52:28.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:28 smithi183 ceph-mon[27058]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-04-02T14:52:28.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:28 smithi122 ceph-mon[24801]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-04-02T14:52:29.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:52:29.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/trash_purge_schedule"}]: dispatch 2024-04-02T14:52:29.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:52:29.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/trash_purge_schedule"}]: dispatch 2024-04-02T14:52:30.513 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:30 smithi183 ceph-mon[27058]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-02T14:52:30.513 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:30 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-02T14:52:30.513 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:30 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:52:30.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:30 smithi122 ceph-mon[24801]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-02T14:52:30.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-02T14:52:30.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:52:31.375 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:31 smithi183 ceph-mon[27058]: Deploying daemon osd.7 on smithi183 2024-04-02T14:52:31.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:31 smithi122 ceph-mon[24801]: Deploying daemon osd.7 on smithi183 2024-04-02T14:52:32.496 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:32 smithi183 ceph-mon[27058]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-02T14:52:32.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:32 smithi122 ceph-mon[24801]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-02T14:52:34.532 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:34 smithi183 ceph-mon[27058]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T14:52:34.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:34 smithi122 ceph-mon[24801]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T14:52:36.064 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:35 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:36.065 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:35 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:52:36.065 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:35 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:52:36.065 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:35 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:52:36.065 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:35 smithi183 ceph-mon[27058]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T14:52:36.088 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:35 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:36.089 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:35 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:52:36.089 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:35 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:52:36.089 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:35 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:52:36.089 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:35 smithi122 ceph-mon[24801]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T14:52:37.540 INFO:teuthology.orchestra.run.smithi183.stdout:Created osd(s) 7 on host 'smithi183' 2024-04-02T14:52:38.399 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:38 smithi183 ceph-mon[27058]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T14:52:38.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:38 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:38.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:38 smithi183 ceph-mon[27058]: from='osd.7 [v2:172.21.15.183:6824/782536404,v1:172.21.15.183:6825/782536404]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T14:52:38.400 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:38 smithi183 ceph-mon[27058]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T14:52:38.445 DEBUG:teuthology.orchestra.run.smithi183:osd.7> sudo journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.7.service 2024-04-02T14:52:38.447 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-02T14:52:38.447 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph osd stat -f json 2024-04-02T14:52:38.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:38 smithi122 ceph-mon[24801]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T14:52:38.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:38 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:38.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:38 smithi122 ceph-mon[24801]: from='osd.7 [v2:172.21.15.183:6824/782536404,v1:172.21.15.183:6825/782536404]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T14:52:38.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:38 smithi122 ceph-mon[24801]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T14:52:39.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:39 smithi183 ceph-mon[27058]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-02T14:52:39.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:39 smithi183 ceph-mon[27058]: osdmap e54: 8 total, 7 up, 8 in 2024-04-02T14:52:39.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:39 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T14:52:39.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:39 smithi183 ceph-mon[27058]: from='osd.7 [v2:172.21.15.183:6824/782536404,v1:172.21.15.183:6825/782536404]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]: dispatch 2024-04-02T14:52:39.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:39 smithi183 ceph-mon[27058]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]: dispatch 2024-04-02T14:52:39.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:39 smithi183 ceph-mon[27058]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T14:52:39.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:39 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:39.651 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:52:39 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:52:39.561+0000 7f4e10bd8700 -1 osd.7 0 waiting for initial osdmap 2024-04-02T14:52:39.651 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:52:39 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:52:39.568+0000 7f4e0a56d700 -1 osd.7 55 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T14:52:39.801 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:39 smithi122 ceph-mon[24801]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-02T14:52:39.802 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:39 smithi122 ceph-mon[24801]: osdmap e54: 8 total, 7 up, 8 in 2024-04-02T14:52:39.802 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:39 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T14:52:39.802 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:39 smithi122 ceph-mon[24801]: from='osd.7 [v2:172.21.15.183:6824/782536404,v1:172.21.15.183:6825/782536404]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]: dispatch 2024-04-02T14:52:39.802 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:39 smithi122 ceph-mon[24801]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]: dispatch 2024-04-02T14:52:39.802 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:39 smithi122 ceph-mon[24801]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T14:52:39.802 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:39 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:40.450 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:52:40.828 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:40 smithi183 ceph-mon[27058]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]': finished 2024-04-02T14:52:40.828 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:40 smithi183 ceph-mon[27058]: osdmap e55: 8 total, 7 up, 8 in 2024-04-02T14:52:40.828 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:40 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T14:52:40.828 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:40 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T14:52:40.828 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:40 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/2103820132' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T14:52:40.828 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:40 smithi183 ceph-mon[27058]: osd.7 [v2:172.21.15.183:6824/782536404,v1:172.21.15.183:6825/782536404] boot 2024-04-02T14:52:40.829 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:40 smithi183 ceph-mon[27058]: osdmap e56: 8 total, 8 up, 8 in 2024-04-02T14:52:40.829 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:40 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T14:52:40.835 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:40 smithi122 ceph-mon[24801]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi183", "root=default"]}]': finished 2024-04-02T14:52:40.835 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:40 smithi122 ceph-mon[24801]: osdmap e55: 8 total, 7 up, 8 in 2024-04-02T14:52:40.835 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:40 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T14:52:40.835 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:40 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T14:52:40.835 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:40 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/2103820132' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T14:52:40.835 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:40 smithi122 ceph-mon[24801]: osd.7 [v2:172.21.15.183:6824/782536404,v1:172.21.15.183:6825/782536404] boot 2024-04-02T14:52:40.836 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:40 smithi122 ceph-mon[24801]: osdmap e56: 8 total, 8 up, 8 in 2024-04-02T14:52:40.836 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:40 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T14:52:41.268 INFO:teuthology.orchestra.run.smithi122.stdout:{"epoch":55,"num_osds":8,"num_up_osds":7,"osd_up_since":1712069536,"num_in_osds":8,"osd_in_since":1712069543,"num_remapped_pgs":0} 2024-04-02T14:52:41.817 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:41 smithi183 ceph-mon[27058]: purged_snaps scrub starts 2024-04-02T14:52:41.817 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:41 smithi183 ceph-mon[27058]: purged_snaps scrub ok 2024-04-02T14:52:41.817 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:41 smithi183 ceph-mon[27058]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:52:41.817 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:41 smithi183 ceph-mon[27058]: osdmap e57: 8 total, 8 up, 8 in 2024-04-02T14:52:41.838 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:41 smithi122 ceph-mon[24801]: purged_snaps scrub starts 2024-04-02T14:52:41.838 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:41 smithi122 ceph-mon[24801]: purged_snaps scrub ok 2024-04-02T14:52:41.838 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:41 smithi122 ceph-mon[24801]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:52:41.838 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:41 smithi122 ceph-mon[24801]: osdmap e57: 8 total, 8 up, 8 in 2024-04-02T14:52:42.270 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph osd stat -f json 2024-04-02T14:52:43.711 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:43 smithi122 ceph-mon[24801]: Detected new or changed devices on smithi183 2024-04-02T14:52:43.711 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:43.711 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:52:43.711 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:52:43.711 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:52:43.711 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:52:43.711 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:43 smithi122 ceph-mon[24801]: Adjusting osd_memory_target on smithi183 to 3984M 2024-04-02T14:52:43.711 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:43.712 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:43.712 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:52:43.712 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:52:43.712 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:52:43.712 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:43.712 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:43 smithi122 ceph-mon[24801]: osdmap e58: 8 total, 8 up, 8 in 2024-04-02T14:52:43.712 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:43 smithi122 ceph-mon[24801]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:52:43.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:43 smithi183 ceph-mon[27058]: Detected new or changed devices on smithi183 2024-04-02T14:52:43.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:43.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:52:43.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:52:43.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:52:43.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-02T14:52:43.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:43 smithi183 ceph-mon[27058]: Adjusting osd_memory_target on smithi183 to 3984M 2024-04-02T14:52:43.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:43.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:43.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:52:43.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:52:43.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:52:43.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:52:43.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:43 smithi183 ceph-mon[27058]: osdmap e58: 8 total, 8 up, 8 in 2024-04-02T14:52:43.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:43 smithi183 ceph-mon[27058]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:52:44.211 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:52:44.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:44 smithi122 ceph-mon[24801]: osdmap e59: 8 total, 8 up, 8 in 2024-04-02T14:52:44.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:44 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/1925235872' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T14:52:44.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:44 smithi183 ceph-mon[27058]: osdmap e59: 8 total, 8 up, 8 in 2024-04-02T14:52:44.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:44 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/1925235872' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T14:52:44.981 INFO:teuthology.orchestra.run.smithi122.stdout:{"epoch":59,"num_osds":8,"num_up_osds":8,"osd_up_since":1712069560,"num_in_osds":8,"osd_in_since":1712069543,"num_remapped_pgs":0} 2024-04-02T14:52:44.982 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-02T14:52:45.195 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:52:45.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:45 smithi122 ceph-mon[24801]: osdmap e60: 8 total, 8 up, 8 in 2024-04-02T14:52:45.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:45 smithi122 ceph-mon[24801]: pgmap v176: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-02T14:52:45.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:45 smithi183 ceph-mon[27058]: osdmap e60: 8 total, 8 up, 8 in 2024-04-02T14:52:45.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:45 smithi183 ceph-mon[27058]: pgmap v176: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-02T14:52:46.960 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:52:46.960 INFO:teuthology.orchestra.run.smithi122.stdout:{"epoch":60,"fsid":"bc643104-f0ff-11ee-b647-cb9ed24678a4","created":"2024-04-02T14:46:40.135303+0000","modified":"2024-04-02T14:52:44.560633+0000","last_up_change":"2024-04-02T14:52:40.550505+0000","last_in_change":"2024-04-02T14:52:23.507167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-02T14:50:41.478782+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"4f45f72f-a976-4503-986c-e03c797072b9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6802","nonce":418135664},{"type":"v1","addr":"172.21.15.122:6803","nonce":418135664}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6804","nonce":418135664},{"type":"v1","addr":"172.21.15.122:6805","nonce":418135664}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6808","nonce":418135664},{"type":"v1","addr":"172.21.15.122:6809","nonce":418135664}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6806","nonce":418135664},{"type":"v1","addr":"172.21.15.122:6807","nonce":418135664}]},"public_addr":"172.21.15.122:6803/418135664","cluster_addr":"172.21.15.122:6805/418135664","heartbeat_back_addr":"172.21.15.122:6809/418135664","heartbeat_front_addr":"172.21.15.122:6807/418135664","state":["exists","up"]},{"osd":1,"uuid":"796f1929-36a0-43ae-8fc9-a02fcb40699a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6810","nonce":2295878653},{"type":"v1","addr":"172.21.15.122:6811","nonce":2295878653}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6812","nonce":2295878653},{"type":"v1","addr":"172.21.15.122:6813","nonce":2295878653}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6816","nonce":2295878653},{"type":"v1","addr":"172.21.15.122:6817","nonce":2295878653}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6814","nonce":2295878653},{"type":"v1","addr":"172.21.15.122:6815","nonce":2295878653}]},"public_addr":"172.21.15.122:6811/2295878653","cluster_addr":"172.21.15.122:6813/2295878653","heartbeat_back_addr":"172.21.15.122:6817/2295878653","heartbeat_front_addr":"172.21.15.122:6815/2295878653","state":["exists","up"]},{"osd":2,"uuid":"369ff431-5778-4d3a-9c18-c4e34b14a540","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6818","nonce":1200971619},{"type":"v1","addr":"172.21.15.122:6819","nonce":1200971619}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6820","nonce":1200971619},{"type":"v1","addr":"172.21.15.122:6821","nonce":1200971619}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6824","nonce":1200971619},{"type":"v1","addr":"172.21.15.122:6825","nonce":1200971619}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6822","nonce":1200971619},{"type":"v1","addr":"172.21.15.122:6823","nonce":1200971619}]},"public_addr":"172.21.15.122:6819/1200971619","cluster_addr":"172.21.15.122:6821/1200971619","heartbeat_back_addr":"172.21.15.122:6825/1200971619","heartbeat_front_addr":"172.21.15.122:6823/1200971619","state":["exists","up"]},{"osd":3,"uuid":"135c10c3-46ab-48b1-967e-fdcfc72eaaa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6826","nonce":685608550},{"type":"v1","addr":"172.21.15.122:6827","nonce":685608550}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6828","nonce":685608550},{"type":"v1","addr":"172.21.15.122:6829","nonce":685608550}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6832","nonce":685608550},{"type":"v1","addr":"172.21.15.122:6833","nonce":685608550}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6830","nonce":685608550},{"type":"v1","addr":"172.21.15.122:6831","nonce":685608550}]},"public_addr":"172.21.15.122:6827/685608550","cluster_addr":"172.21.15.122:6829/685608550","heartbeat_back_addr":"172.21.15.122:6833/685608550","heartbeat_front_addr":"172.21.15.122:6831/685608550","state":["exists","up"]},{"osd":4,"uuid":"d3a1fe32-a760-498b-94a3-a37e342ec4db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6800","nonce":3537677704},{"type":"v1","addr":"172.21.15.183:6801","nonce":3537677704}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6802","nonce":3537677704},{"type":"v1","addr":"172.21.15.183:6803","nonce":3537677704}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6806","nonce":3537677704},{"type":"v1","addr":"172.21.15.183:6807","nonce":3537677704}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6804","nonce":3537677704},{"type":"v1","addr":"172.21.15.183:6805","nonce":3537677704}]},"public_addr":"172.21.15.183:6801/3537677704","cluster_addr":"172.21.15.183:6803/3537677704","heartbeat_back_addr":"172.21.15.183:6807/3537677704","heartbeat_front_addr":"172.21.15.183:6805/3537677704","state":["exists","up"]},{"osd":5,"uuid":"9393d535-0a63-462f-9db6-a461b1c21dbe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6808","nonce":273662903},{"type":"v1","addr":"172.21.15.183:6809","nonce":273662903}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6810","nonce":273662903},{"type":"v1","addr":"172.21.15.183:6811","nonce":273662903}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6814","nonce":273662903},{"type":"v1","addr":"172.21.15.183:6815","nonce":273662903}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6812","nonce":273662903},{"type":"v1","addr":"172.21.15.183:6813","nonce":273662903}]},"public_addr":"172.21.15.183:6809/273662903","cluster_addr":"172.21.15.183:6811/273662903","heartbeat_back_addr":"172.21.15.183:6815/273662903","heartbeat_front_addr":"172.21.15.183:6813/273662903","state":["exists","up"]},{"osd":6,"uuid":"4529b54e-730d-4036-b885-da2b4770c7d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6816","nonce":3624098587},{"type":"v1","addr":"172.21.15.183:6817","nonce":3624098587}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6818","nonce":3624098587},{"type":"v1","addr":"172.21.15.183:6819","nonce":3624098587}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6822","nonce":3624098587},{"type":"v1","addr":"172.21.15.183:6823","nonce":3624098587}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6820","nonce":3624098587},{"type":"v1","addr":"172.21.15.183:6821","nonce":3624098587}]},"public_addr":"172.21.15.183:6817/3624098587","cluster_addr":"172.21.15.183:6819/3624098587","heartbeat_back_addr":"172.21.15.183:6823/3624098587","heartbeat_front_addr":"172.21.15.183:6821/3624098587","state":["exists","up"]},{"osd":7,"uuid":"cabca570-66f6-4c9d-b8e2-5cc250be64e4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6824","nonce":782536404},{"type":"v1","addr":"172.21.15.183:6825","nonce":782536404}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6826","nonce":782536404},{"type":"v1","addr":"172.21.15.183:6827","nonce":782536404}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6830","nonce":782536404},{"type":"v1","addr":"172.21.15.183:6831","nonce":782536404}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6828","nonce":782536404},{"type":"v1","addr":"172.21.15.183:6829","nonce":782536404}]},"public_addr":"172.21.15.183:6825/782536404","cluster_addr":"172.21.15.183:6827/782536404","heartbeat_back_addr":"172.21.15.183:6831/782536404","heartbeat_front_addr":"172.21.15.183:6829/782536404","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:49:36.100672+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:50:09.865970+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:50:38.230894+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:51:04.246159+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:51:26.761765+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:51:50.434907+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:52:14.494498+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:52:38.812730+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.122:0/3214361776":"2024-04-03T14:48:29.116730+0000","172.21.15.122:6801/909137736":"2024-04-03T14:48:29.116730+0000","172.21.15.122:6800/909137736":"2024-04-03T14:48:29.116730+0000","172.21.15.122:0/349692880":"2024-04-03T14:47:36.546336+0000","172.21.15.122:6800/3227488636":"2024-04-03T14:47:36.546336+0000","172.21.15.122:0/394707067":"2024-04-03T14:47:36.546336+0000","172.21.15.122:6801/3227488636":"2024-04-03T14:47:36.546336+0000","172.21.15.122:0/2359725635":"2024-04-03T14:47:07.235359+0000","172.21.15.122:0/1083107235":"2024-04-03T14:48:29.116730+0000","172.21.15.122:0/3218850064":"2024-04-03T14:47:36.546336+0000","172.21.15.122:0/635302770":"2024-04-03T14:47:07.235359+0000","172.21.15.122:6800/4275247818":"2024-04-03T14:47:07.235359+0000","172.21.15.122:0/3387876048":"2024-04-03T14:47:07.235359+0000","172.21.15.122:0/3367066727":"2024-04-03T14:48:29.116730+0000","172.21.15.122:6801/4275247818":"2024-04-03T14:47:07.235359+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-02T14:52:47.256 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:47 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/2755720394' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T14:52:47.399 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:47 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/2755720394' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T14:52:47.655 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-02T14:50:41.478782+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-02T14:52:47.656 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph osd pool get .mgr pg_num 2024-04-02T14:52:47.870 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:52:48.334 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:48 smithi122 ceph-mon[24801]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-02T14:52:48.399 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:48 smithi183 ceph-mon[27058]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-02T14:52:49.683 INFO:teuthology.orchestra.run.smithi122.stdout:pg_num: 1 2024-04-02T14:52:50.401 INFO:tasks.cephadm:Setting up client nodes... 2024-04-02T14:52:50.401 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-02T14:52:50.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:50 smithi122 ceph-mon[24801]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-04-02T14:52:50.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:50 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/1583049408' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-02T14:52:50.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:50 smithi183 ceph-mon[27058]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-04-02T14:52:50.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:50 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/1583049408' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-02T14:52:52.338 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:52 smithi122 ceph-mon[24801]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-02T14:52:52.541 INFO:teuthology.orchestra.run.smithi122.stdout:[client.0] 2024-04-02T14:52:52.542 INFO:teuthology.orchestra.run.smithi122.stdout: key = AQDEGwxmA2+eIBAAoaWEfXXgDQdPQyk+g79acA== 2024-04-02T14:52:52.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:52 smithi183 ceph-mon[27058]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-02T14:52:53.156 DEBUG:teuthology.orchestra.run.smithi122:> set -ex 2024-04-02T14:52:53.156 DEBUG:teuthology.orchestra.run.smithi122:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-02T14:52:53.156 DEBUG:teuthology.orchestra.run.smithi122:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-02T14:52:53.199 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-02T14:52:53.199 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-02T14:52:53.199 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph mgr dump --format=json 2024-04-02T14:52:53.450 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:52:53.572 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:53 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/3565500594' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T14:52:53.572 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:53 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/3565500594' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T14:52:53.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:53 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/3565500594' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T14:52:53.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:53 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/3565500594' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T14:52:54.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:54 smithi122 ceph-mon[24801]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:52:54.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:54 smithi183 ceph-mon[27058]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:52:55.315 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:52:55.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:55 smithi122 ceph-mon[24801]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:52:55.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:55 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/1256902899' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T14:52:55.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:55 smithi183 ceph-mon[27058]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:52:55.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:55 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/1256902899' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T14:52:55.928 INFO:teuthology.orchestra.run.smithi122.stdout:{"epoch":19,"active_gid":14200,"active_name":"smithi122.szjkfg","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6800","nonce":1421596908},{"type":"v1","addr":"172.21.15.122:6801","nonce":1421596908}]},"active_addr":"172.21.15.122:6801/1421596908","active_change":"2024-04-02T14:48:29.117057+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14220,"name":"smithi183.smxedq","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.122:8443/","prometheus":"http://172.21.15.122:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.122:0","nonce":2431026485}]},{"addrvec":[{"type":"v2","addr":"172.21.15.122:0","nonce":252811337}]},{"addrvec":[{"type":"v2","addr":"172.21.15.122:0","nonce":2597381754}]},{"addrvec":[{"type":"v2","addr":"172.21.15.122:0","nonce":1375979845}]}]}} 2024-04-02T14:52:55.933 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-02T14:52:55.933 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-02T14:52:55.934 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-02T14:52:56.145 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:52:58.014 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:52:58.014 INFO:teuthology.orchestra.run.smithi122.stdout:{"epoch":60,"fsid":"bc643104-f0ff-11ee-b647-cb9ed24678a4","created":"2024-04-02T14:46:40.135303+0000","modified":"2024-04-02T14:52:44.560633+0000","last_up_change":"2024-04-02T14:52:40.550505+0000","last_in_change":"2024-04-02T14:52:23.507167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-02T14:50:41.478782+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"4f45f72f-a976-4503-986c-e03c797072b9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6802","nonce":418135664},{"type":"v1","addr":"172.21.15.122:6803","nonce":418135664}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6804","nonce":418135664},{"type":"v1","addr":"172.21.15.122:6805","nonce":418135664}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6808","nonce":418135664},{"type":"v1","addr":"172.21.15.122:6809","nonce":418135664}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6806","nonce":418135664},{"type":"v1","addr":"172.21.15.122:6807","nonce":418135664}]},"public_addr":"172.21.15.122:6803/418135664","cluster_addr":"172.21.15.122:6805/418135664","heartbeat_back_addr":"172.21.15.122:6809/418135664","heartbeat_front_addr":"172.21.15.122:6807/418135664","state":["exists","up"]},{"osd":1,"uuid":"796f1929-36a0-43ae-8fc9-a02fcb40699a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6810","nonce":2295878653},{"type":"v1","addr":"172.21.15.122:6811","nonce":2295878653}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6812","nonce":2295878653},{"type":"v1","addr":"172.21.15.122:6813","nonce":2295878653}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6816","nonce":2295878653},{"type":"v1","addr":"172.21.15.122:6817","nonce":2295878653}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6814","nonce":2295878653},{"type":"v1","addr":"172.21.15.122:6815","nonce":2295878653}]},"public_addr":"172.21.15.122:6811/2295878653","cluster_addr":"172.21.15.122:6813/2295878653","heartbeat_back_addr":"172.21.15.122:6817/2295878653","heartbeat_front_addr":"172.21.15.122:6815/2295878653","state":["exists","up"]},{"osd":2,"uuid":"369ff431-5778-4d3a-9c18-c4e34b14a540","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6818","nonce":1200971619},{"type":"v1","addr":"172.21.15.122:6819","nonce":1200971619}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6820","nonce":1200971619},{"type":"v1","addr":"172.21.15.122:6821","nonce":1200971619}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6824","nonce":1200971619},{"type":"v1","addr":"172.21.15.122:6825","nonce":1200971619}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6822","nonce":1200971619},{"type":"v1","addr":"172.21.15.122:6823","nonce":1200971619}]},"public_addr":"172.21.15.122:6819/1200971619","cluster_addr":"172.21.15.122:6821/1200971619","heartbeat_back_addr":"172.21.15.122:6825/1200971619","heartbeat_front_addr":"172.21.15.122:6823/1200971619","state":["exists","up"]},{"osd":3,"uuid":"135c10c3-46ab-48b1-967e-fdcfc72eaaa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6826","nonce":685608550},{"type":"v1","addr":"172.21.15.122:6827","nonce":685608550}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6828","nonce":685608550},{"type":"v1","addr":"172.21.15.122:6829","nonce":685608550}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6832","nonce":685608550},{"type":"v1","addr":"172.21.15.122:6833","nonce":685608550}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6830","nonce":685608550},{"type":"v1","addr":"172.21.15.122:6831","nonce":685608550}]},"public_addr":"172.21.15.122:6827/685608550","cluster_addr":"172.21.15.122:6829/685608550","heartbeat_back_addr":"172.21.15.122:6833/685608550","heartbeat_front_addr":"172.21.15.122:6831/685608550","state":["exists","up"]},{"osd":4,"uuid":"d3a1fe32-a760-498b-94a3-a37e342ec4db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6800","nonce":3537677704},{"type":"v1","addr":"172.21.15.183:6801","nonce":3537677704}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6802","nonce":3537677704},{"type":"v1","addr":"172.21.15.183:6803","nonce":3537677704}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6806","nonce":3537677704},{"type":"v1","addr":"172.21.15.183:6807","nonce":3537677704}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6804","nonce":3537677704},{"type":"v1","addr":"172.21.15.183:6805","nonce":3537677704}]},"public_addr":"172.21.15.183:6801/3537677704","cluster_addr":"172.21.15.183:6803/3537677704","heartbeat_back_addr":"172.21.15.183:6807/3537677704","heartbeat_front_addr":"172.21.15.183:6805/3537677704","state":["exists","up"]},{"osd":5,"uuid":"9393d535-0a63-462f-9db6-a461b1c21dbe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6808","nonce":273662903},{"type":"v1","addr":"172.21.15.183:6809","nonce":273662903}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6810","nonce":273662903},{"type":"v1","addr":"172.21.15.183:6811","nonce":273662903}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6814","nonce":273662903},{"type":"v1","addr":"172.21.15.183:6815","nonce":273662903}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6812","nonce":273662903},{"type":"v1","addr":"172.21.15.183:6813","nonce":273662903}]},"public_addr":"172.21.15.183:6809/273662903","cluster_addr":"172.21.15.183:6811/273662903","heartbeat_back_addr":"172.21.15.183:6815/273662903","heartbeat_front_addr":"172.21.15.183:6813/273662903","state":["exists","up"]},{"osd":6,"uuid":"4529b54e-730d-4036-b885-da2b4770c7d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6816","nonce":3624098587},{"type":"v1","addr":"172.21.15.183:6817","nonce":3624098587}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6818","nonce":3624098587},{"type":"v1","addr":"172.21.15.183:6819","nonce":3624098587}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6822","nonce":3624098587},{"type":"v1","addr":"172.21.15.183:6823","nonce":3624098587}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6820","nonce":3624098587},{"type":"v1","addr":"172.21.15.183:6821","nonce":3624098587}]},"public_addr":"172.21.15.183:6817/3624098587","cluster_addr":"172.21.15.183:6819/3624098587","heartbeat_back_addr":"172.21.15.183:6823/3624098587","heartbeat_front_addr":"172.21.15.183:6821/3624098587","state":["exists","up"]},{"osd":7,"uuid":"cabca570-66f6-4c9d-b8e2-5cc250be64e4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6824","nonce":782536404},{"type":"v1","addr":"172.21.15.183:6825","nonce":782536404}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6826","nonce":782536404},{"type":"v1","addr":"172.21.15.183:6827","nonce":782536404}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6830","nonce":782536404},{"type":"v1","addr":"172.21.15.183:6831","nonce":782536404}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6828","nonce":782536404},{"type":"v1","addr":"172.21.15.183:6829","nonce":782536404}]},"public_addr":"172.21.15.183:6825/782536404","cluster_addr":"172.21.15.183:6827/782536404","heartbeat_back_addr":"172.21.15.183:6831/782536404","heartbeat_front_addr":"172.21.15.183:6829/782536404","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:49:36.100672+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:50:09.865970+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:50:38.230894+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:51:04.246159+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:51:26.761765+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:51:50.434907+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:52:14.494498+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:52:38.812730+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.122:0/3214361776":"2024-04-03T14:48:29.116730+0000","172.21.15.122:6801/909137736":"2024-04-03T14:48:29.116730+0000","172.21.15.122:6800/909137736":"2024-04-03T14:48:29.116730+0000","172.21.15.122:0/349692880":"2024-04-03T14:47:36.546336+0000","172.21.15.122:6800/3227488636":"2024-04-03T14:47:36.546336+0000","172.21.15.122:0/394707067":"2024-04-03T14:47:36.546336+0000","172.21.15.122:6801/3227488636":"2024-04-03T14:47:36.546336+0000","172.21.15.122:0/2359725635":"2024-04-03T14:47:07.235359+0000","172.21.15.122:0/1083107235":"2024-04-03T14:48:29.116730+0000","172.21.15.122:0/3218850064":"2024-04-03T14:47:36.546336+0000","172.21.15.122:0/635302770":"2024-04-03T14:47:07.235359+0000","172.21.15.122:6800/4275247818":"2024-04-03T14:47:07.235359+0000","172.21.15.122:0/3387876048":"2024-04-03T14:47:07.235359+0000","172.21.15.122:0/3367066727":"2024-04-03T14:48:29.116730+0000","172.21.15.122:6801/4275247818":"2024-04-03T14:47:07.235359+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-02T14:52:58.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:58 smithi122 ceph-mon[24801]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:52:58.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:52:58 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/2656435338' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T14:52:58.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:58 smithi183 ceph-mon[27058]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:52:58.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:52:58 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/2656435338' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T14:52:58.746 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-02T14:52:58.746 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-02T14:52:58.962 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:00.311 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:00 smithi122 ceph-mon[24801]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:00.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:00 smithi183 ceph-mon[27058]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:00.693 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:53:00.693 INFO:teuthology.orchestra.run.smithi122.stdout:{"epoch":60,"fsid":"bc643104-f0ff-11ee-b647-cb9ed24678a4","created":"2024-04-02T14:46:40.135303+0000","modified":"2024-04-02T14:52:44.560633+0000","last_up_change":"2024-04-02T14:52:40.550505+0000","last_in_change":"2024-04-02T14:52:23.507167+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-02T14:50:41.478782+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"4f45f72f-a976-4503-986c-e03c797072b9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6802","nonce":418135664},{"type":"v1","addr":"172.21.15.122:6803","nonce":418135664}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6804","nonce":418135664},{"type":"v1","addr":"172.21.15.122:6805","nonce":418135664}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6808","nonce":418135664},{"type":"v1","addr":"172.21.15.122:6809","nonce":418135664}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6806","nonce":418135664},{"type":"v1","addr":"172.21.15.122:6807","nonce":418135664}]},"public_addr":"172.21.15.122:6803/418135664","cluster_addr":"172.21.15.122:6805/418135664","heartbeat_back_addr":"172.21.15.122:6809/418135664","heartbeat_front_addr":"172.21.15.122:6807/418135664","state":["exists","up"]},{"osd":1,"uuid":"796f1929-36a0-43ae-8fc9-a02fcb40699a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6810","nonce":2295878653},{"type":"v1","addr":"172.21.15.122:6811","nonce":2295878653}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6812","nonce":2295878653},{"type":"v1","addr":"172.21.15.122:6813","nonce":2295878653}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6816","nonce":2295878653},{"type":"v1","addr":"172.21.15.122:6817","nonce":2295878653}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6814","nonce":2295878653},{"type":"v1","addr":"172.21.15.122:6815","nonce":2295878653}]},"public_addr":"172.21.15.122:6811/2295878653","cluster_addr":"172.21.15.122:6813/2295878653","heartbeat_back_addr":"172.21.15.122:6817/2295878653","heartbeat_front_addr":"172.21.15.122:6815/2295878653","state":["exists","up"]},{"osd":2,"uuid":"369ff431-5778-4d3a-9c18-c4e34b14a540","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6818","nonce":1200971619},{"type":"v1","addr":"172.21.15.122:6819","nonce":1200971619}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6820","nonce":1200971619},{"type":"v1","addr":"172.21.15.122:6821","nonce":1200971619}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6824","nonce":1200971619},{"type":"v1","addr":"172.21.15.122:6825","nonce":1200971619}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6822","nonce":1200971619},{"type":"v1","addr":"172.21.15.122:6823","nonce":1200971619}]},"public_addr":"172.21.15.122:6819/1200971619","cluster_addr":"172.21.15.122:6821/1200971619","heartbeat_back_addr":"172.21.15.122:6825/1200971619","heartbeat_front_addr":"172.21.15.122:6823/1200971619","state":["exists","up"]},{"osd":3,"uuid":"135c10c3-46ab-48b1-967e-fdcfc72eaaa4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6826","nonce":685608550},{"type":"v1","addr":"172.21.15.122:6827","nonce":685608550}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6828","nonce":685608550},{"type":"v1","addr":"172.21.15.122:6829","nonce":685608550}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6832","nonce":685608550},{"type":"v1","addr":"172.21.15.122:6833","nonce":685608550}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6830","nonce":685608550},{"type":"v1","addr":"172.21.15.122:6831","nonce":685608550}]},"public_addr":"172.21.15.122:6827/685608550","cluster_addr":"172.21.15.122:6829/685608550","heartbeat_back_addr":"172.21.15.122:6833/685608550","heartbeat_front_addr":"172.21.15.122:6831/685608550","state":["exists","up"]},{"osd":4,"uuid":"d3a1fe32-a760-498b-94a3-a37e342ec4db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6800","nonce":3537677704},{"type":"v1","addr":"172.21.15.183:6801","nonce":3537677704}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6802","nonce":3537677704},{"type":"v1","addr":"172.21.15.183:6803","nonce":3537677704}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6806","nonce":3537677704},{"type":"v1","addr":"172.21.15.183:6807","nonce":3537677704}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6804","nonce":3537677704},{"type":"v1","addr":"172.21.15.183:6805","nonce":3537677704}]},"public_addr":"172.21.15.183:6801/3537677704","cluster_addr":"172.21.15.183:6803/3537677704","heartbeat_back_addr":"172.21.15.183:6807/3537677704","heartbeat_front_addr":"172.21.15.183:6805/3537677704","state":["exists","up"]},{"osd":5,"uuid":"9393d535-0a63-462f-9db6-a461b1c21dbe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6808","nonce":273662903},{"type":"v1","addr":"172.21.15.183:6809","nonce":273662903}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6810","nonce":273662903},{"type":"v1","addr":"172.21.15.183:6811","nonce":273662903}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6814","nonce":273662903},{"type":"v1","addr":"172.21.15.183:6815","nonce":273662903}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6812","nonce":273662903},{"type":"v1","addr":"172.21.15.183:6813","nonce":273662903}]},"public_addr":"172.21.15.183:6809/273662903","cluster_addr":"172.21.15.183:6811/273662903","heartbeat_back_addr":"172.21.15.183:6815/273662903","heartbeat_front_addr":"172.21.15.183:6813/273662903","state":["exists","up"]},{"osd":6,"uuid":"4529b54e-730d-4036-b885-da2b4770c7d5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6816","nonce":3624098587},{"type":"v1","addr":"172.21.15.183:6817","nonce":3624098587}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6818","nonce":3624098587},{"type":"v1","addr":"172.21.15.183:6819","nonce":3624098587}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6822","nonce":3624098587},{"type":"v1","addr":"172.21.15.183:6823","nonce":3624098587}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6820","nonce":3624098587},{"type":"v1","addr":"172.21.15.183:6821","nonce":3624098587}]},"public_addr":"172.21.15.183:6817/3624098587","cluster_addr":"172.21.15.183:6819/3624098587","heartbeat_back_addr":"172.21.15.183:6823/3624098587","heartbeat_front_addr":"172.21.15.183:6821/3624098587","state":["exists","up"]},{"osd":7,"uuid":"cabca570-66f6-4c9d-b8e2-5cc250be64e4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6824","nonce":782536404},{"type":"v1","addr":"172.21.15.183:6825","nonce":782536404}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6826","nonce":782536404},{"type":"v1","addr":"172.21.15.183:6827","nonce":782536404}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6830","nonce":782536404},{"type":"v1","addr":"172.21.15.183:6831","nonce":782536404}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6828","nonce":782536404},{"type":"v1","addr":"172.21.15.183:6829","nonce":782536404}]},"public_addr":"172.21.15.183:6825/782536404","cluster_addr":"172.21.15.183:6827/782536404","heartbeat_back_addr":"172.21.15.183:6831/782536404","heartbeat_front_addr":"172.21.15.183:6829/782536404","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:49:36.100672+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:50:09.865970+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:50:38.230894+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:51:04.246159+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:51:26.761765+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:51:50.434907+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:52:14.494498+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T14:52:38.812730+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.122:0/3214361776":"2024-04-03T14:48:29.116730+0000","172.21.15.122:6801/909137736":"2024-04-03T14:48:29.116730+0000","172.21.15.122:6800/909137736":"2024-04-03T14:48:29.116730+0000","172.21.15.122:0/349692880":"2024-04-03T14:47:36.546336+0000","172.21.15.122:6800/3227488636":"2024-04-03T14:47:36.546336+0000","172.21.15.122:0/394707067":"2024-04-03T14:47:36.546336+0000","172.21.15.122:6801/3227488636":"2024-04-03T14:47:36.546336+0000","172.21.15.122:0/2359725635":"2024-04-03T14:47:07.235359+0000","172.21.15.122:0/1083107235":"2024-04-03T14:48:29.116730+0000","172.21.15.122:0/3218850064":"2024-04-03T14:47:36.546336+0000","172.21.15.122:0/635302770":"2024-04-03T14:47:07.235359+0000","172.21.15.122:6800/4275247818":"2024-04-03T14:47:07.235359+0000","172.21.15.122:0/3387876048":"2024-04-03T14:47:07.235359+0000","172.21.15.122:0/3367066727":"2024-04-03T14:48:29.116730+0000","172.21.15.122:6801/4275247818":"2024-04-03T14:47:07.235359+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-02T14:53:01.478 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:01 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/3434377931' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T14:53:01.583 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph tell osd.0 flush_pg_stats 2024-04-02T14:53:01.583 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph tell osd.1 flush_pg_stats 2024-04-02T14:53:01.584 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph tell osd.2 flush_pg_stats 2024-04-02T14:53:01.584 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph tell osd.3 flush_pg_stats 2024-04-02T14:53:01.584 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph tell osd.4 flush_pg_stats 2024-04-02T14:53:01.585 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph tell osd.5 flush_pg_stats 2024-04-02T14:53:01.585 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph tell osd.6 flush_pg_stats 2024-04-02T14:53:01.586 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph tell osd.7 flush_pg_stats 2024-04-02T14:53:01.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:01 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/3434377931' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T14:53:02.028 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:02.029 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:02.029 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:02.031 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:02.031 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:02.035 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:02.035 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:02.056 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:02.423 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:02 smithi122 ceph-mon[24801]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:02.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:02 smithi183 ceph-mon[27058]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:04.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:04 smithi122 ceph-mon[24801]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:04.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:04 smithi183 ceph-mon[27058]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:05.838 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:05 smithi122 ceph-mon[24801]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:05.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:05 smithi183 ceph-mon[27058]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:08.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:08 smithi122 ceph-mon[24801]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:08.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:08 smithi183 ceph-mon[27058]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:10.461 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:10 smithi122 ceph-mon[24801]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:10.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:10 smithi183 ceph-mon[27058]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:11.757 INFO:teuthology.orchestra.run.smithi122.stdout:240518168584 2024-04-02T14:53:11.758 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.7 2024-04-02T14:53:12.529 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:12 smithi122 ceph-mon[24801]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:12.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:12 smithi183 ceph-mon[27058]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:13.648 INFO:teuthology.orchestra.run.smithi122.stdout:206158430221 2024-04-02T14:53:13.649 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.6 2024-04-02T14:53:14.358 INFO:teuthology.orchestra.run.smithi122.stdout:81604378658 2024-04-02T14:53:14.358 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.2 2024-04-02T14:53:14.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:14 smithi122 ceph-mon[24801]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:14.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:14 smithi183 ceph-mon[27058]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:15.347 INFO:teuthology.orchestra.run.smithi122.stdout:171798691859 2024-04-02T14:53:15.347 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.5 2024-04-02T14:53:15.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:15 smithi122 ceph-mon[24801]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:15.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:15 smithi183 ceph-mon[27058]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:16.447 INFO:teuthology.orchestra.run.smithi122.stdout:137438953495 2024-04-02T14:53:16.447 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.4 2024-04-02T14:53:16.804 INFO:teuthology.orchestra.run.smithi122.stdout:38654705710 2024-04-02T14:53:16.804 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.0 2024-04-02T14:53:16.821 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:16.821 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:16.822 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:18.523 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:18 smithi122 ceph-mon[24801]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:18.602 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:18 smithi183 ceph-mon[27058]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:18.769 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:19.527 INFO:teuthology.orchestra.run.smithi122.stdout:60129542184 2024-04-02T14:53:19.527 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.1 2024-04-02T14:53:20.225 INFO:teuthology.orchestra.run.smithi122.stdout:240518168586 2024-04-02T14:53:20.505 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:20 smithi122 ceph-mon[24801]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:20.505 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:20 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/3054697598' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-02T14:53:20.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:20 smithi183 ceph-mon[27058]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:20.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:20 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/3054697598' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-02T14:53:20.818 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168584 got 240518168586 for osd.7 2024-04-02T14:53:20.818 DEBUG:teuthology.parallel:result is None 2024-04-02T14:53:21.850 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:21.850 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:22.523 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:22 smithi122 ceph-mon[24801]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:22.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:22 smithi183 ceph-mon[27058]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:24.519 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:24 smithi122 ceph-mon[24801]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:24.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:24 smithi183 ceph-mon[27058]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:25.003 INFO:teuthology.orchestra.run.smithi122.stdout:171798691861 2024-04-02T14:53:25.838 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:25 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/374812420' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-02T14:53:25.838 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:25 smithi122 ceph-mon[24801]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:25.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:25 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/374812420' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-02T14:53:25.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:25 smithi183 ceph-mon[27058]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:25.978 INFO:teuthology.orchestra.run.smithi122.stdout:115964117023 2024-04-02T14:53:25.979 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.3 2024-04-02T14:53:26.496 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:26.882 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691859 got 171798691861 for osd.5 2024-04-02T14:53:26.882 DEBUG:teuthology.parallel:result is None 2024-04-02T14:53:27.790 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:27.968 INFO:teuthology.orchestra.run.smithi122.stdout:137438953498 2024-04-02T14:53:28.169 INFO:teuthology.orchestra.run.smithi122.stdout:81604378661 2024-04-02T14:53:28.523 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:28 smithi122 ceph-mon[24801]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:28.523 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:28 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/4125737755' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-02T14:53:28.523 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:28 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/3883945925' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-02T14:53:28.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:28 smithi183 ceph-mon[27058]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:28.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:28 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/4125737755' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-02T14:53:28.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:28 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/3883945925' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-02T14:53:28.730 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378658 got 81604378661 for osd.2 2024-04-02T14:53:28.730 DEBUG:teuthology.parallel:result is None 2024-04-02T14:53:29.535 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:53:29.535 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/trash_purge_schedule"}]: dispatch 2024-04-02T14:53:29.564 INFO:tasks.cephadm.ceph_manager.ceph:need seq 137438953495 got 137438953498 for osd.4 2024-04-02T14:53:29.564 DEBUG:teuthology.parallel:result is None 2024-04-02T14:53:29.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:53:29.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/trash_purge_schedule"}]: dispatch 2024-04-02T14:53:30.520 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:30 smithi122 ceph-mon[24801]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:30.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:30 smithi183 ceph-mon[27058]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:30.964 INFO:teuthology.orchestra.run.smithi122.stdout:38654705714 2024-04-02T14:53:31.534 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:31 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/2158045347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-02T14:53:31.557 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705710 got 38654705714 for osd.0 2024-04-02T14:53:31.557 DEBUG:teuthology.parallel:result is None 2024-04-02T14:53:31.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:31 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/2158045347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-02T14:53:32.532 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:32 smithi122 ceph-mon[24801]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:32.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:32 smithi183 ceph-mon[27058]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:32.813 INFO:teuthology.orchestra.run.smithi122.stdout:206158430225 2024-04-02T14:53:33.425 INFO:teuthology.orchestra.run.smithi122.stdout:115964117025 2024-04-02T14:53:33.426 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:33 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/1679789398' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-02T14:53:33.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:33 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/1679789398' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-02T14:53:33.750 INFO:teuthology.orchestra.run.smithi122.stdout:60129542188 2024-04-02T14:53:34.303 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430221 got 206158430225 for osd.6 2024-04-02T14:53:34.303 DEBUG:teuthology.parallel:result is None 2024-04-02T14:53:34.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:34 smithi122 ceph-mon[24801]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:34.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:34 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/3901055263' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-02T14:53:34.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:34 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/1240492530' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-02T14:53:34.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:34 smithi183 ceph-mon[27058]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:34.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:34 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/3901055263' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-02T14:53:34.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:34 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/1240492530' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-02T14:53:34.740 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542184 got 60129542188 for osd.1 2024-04-02T14:53:34.740 DEBUG:teuthology.parallel:result is None 2024-04-02T14:53:35.101 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117023 got 115964117025 for osd.3 2024-04-02T14:53:35.101 DEBUG:teuthology.parallel:result is None 2024-04-02T14:53:35.101 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-02T14:53:35.101 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-02T14:53:35.324 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:35.778 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:35 smithi122 ceph-mon[24801]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:35.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:35 smithi183 ceph-mon[27058]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:37.020 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:53:37.021 INFO:teuthology.orchestra.run.smithi122.stderr:dumped all 2024-04-02T14:53:37.650 INFO:teuthology.orchestra.run.smithi122.stdout:{"pg_ready":true,"pg_map":{"version":201,"stamp":"2024-04-02T14:53:35.221854+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48808,"kb_used_data":6376,"kb_used_omap":0,"kb_used_meta":42368,"kb_avail":749912408,"statfs":{"total":767960285184,"available":767910305792,"internally_reserved":0,"allocated":6529024,"data_stored":3571378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43384832},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001676"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":222,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-04-02T14:52:44.576111+0000","last_change":"2024-04-02T14:52:44.576111+0000","last_active":"2024-04-02T14:52:44.576111+0000","last_peered":"2024-04-02T14:52:44.576111+0000","last_clean":"2024-04-02T14:52:44.576111+0000","last_became_active":"2024-04-02T14:52:44.575660+0000","last_became_peered":"2024-04-02T14:52:44.575660+0000","last_unstale":"2024-04-02T14:52:44.576111+0000","last_undegraded":"2024-04-02T14:52:44.576111+0000","last_fullsized":"2024-04-02T14:52:44.576111+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-02T14:50:41.534364+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-02T14:50:41.534364+0000","last_clean_scrub_stamp":"2024-04-02T14:50:41.534364+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-04T00:54:34.035198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168589,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776819,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70099999999999996}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57199999999999995}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66100000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.623}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59599999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.495}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63200000000000001}]}]},{"osd":6,"up_from":48,"seq":206158430226,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6184,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738968,"statfs":{"total":95995035648,"available":95988703232,"internally_reserved":0,"allocated":1146880,"data_stored":776877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 14:53:17 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.495}]},{"osd":1,"last update":"Tue Apr 2 14:53:17 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.53600000000000003}]},{"osd":2,"last update":"Tue Apr 2 14:53:17 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.64200000000000002}]},{"osd":3,"last update":"Tue Apr 2 14:53:17 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.67400000000000004}]},{"osd":4,"last update":"Tue Apr 2 14:53:17 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.76400000000000001}]},{"osd":5,"last update":"Tue Apr 2 14:53:17 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.80200000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.749}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 14:53:17 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.34499999999999997,"15min":0.32300000000000001},"min":{"1min":0.26400000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":1.3380000000000001,"5min":1.3380000000000001,"15min":1.3380000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.35599999999999998,"15min":0.33000000000000002},"min":{"1min":0.252,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.498}]},{"osd":2,"last update":"Tue Apr 2 14:52:43 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.38400000000000001,"15min":0.378},"min":{"1min":0.27100000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.442,"5min":0.40500000000000003,"15min":0.39900000000000002},"min":{"1min":0.32900000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.48099999999999998}]},{"osd":3,"last update":"Tue Apr 2 14:53:11 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.435,"15min":0.44},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.46200000000000002}]},{"osd":4,"last update":"Tue Apr 2 14:52:30 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.68200000000000005}]},{"osd":5,"last update":"Tue Apr 2 14:52:56 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.66200000000000003}]},{"osd":6,"last update":"Tue Apr 2 14:53:17 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.64700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.74299999999999999}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Apr 2 14:53:18 2024","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.34399999999999997,"15min":0.33200000000000002},"min":{"1min":0.23000000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.52500000000000002,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.36099999999999999,"5min":0.34999999999999998,"15min":0.34000000000000002},"min":{"1min":0.248,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.307}]},{"osd":2,"last update":"Tue Apr 2 14:52:45 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.40500000000000003,"15min":0.40100000000000002},"min":{"1min":0.29499999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.51200000000000001,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.28799999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.54100000000000004,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.37}]},{"osd":3,"last update":"Tue Apr 2 14:53:11 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44800000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.55300000000000005,"5min":0.622,"15min":0.622},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.372,"5min":0.42599999999999999,"15min":0.435},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.503,"5min":0.626,"15min":0.626},"last":0.33500000000000002}]},{"osd":4,"last update":"Tue Apr 2 14:52:31 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.997,"5min":0.997,"15min":0.997},"last":0.495},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.52400000000000002}]},{"osd":5,"last update":"Tue Apr 2 14:52:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.47499999999999998}]},{"osd":6,"last update":"Tue Apr 2 14:53:18 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.435}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56799999999999995}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6240,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738912,"statfs":{"total":95995035648,"available":95988645888,"internally_reserved":0,"allocated":614400,"data_stored":247949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 14:52:43 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.379,"15min":0.378},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.378,"5min":0.36599999999999999,"15min":0.36399999999999999},"min":{"1min":0.27700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.29899999999999999}]},{"osd":1,"last update":"Tue Apr 2 14:52:43 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.39200000000000002,"15min":0.39500000000000002},"min":{"1min":0.26300000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.41099999999999998,"15min":0.40799999999999997},"min":{"1min":0.29999999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.54600000000000004,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.26600000000000001}]},{"osd":3,"last update":"Tue Apr 2 14:53:10 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.437,"15min":0.441},"min":{"1min":0.318,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.432,"5min":0.45100000000000001,"15min":0.45400000000000001},"min":{"1min":0.311,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.35599999999999998}]},{"osd":4,"last update":"Tue Apr 2 14:52:30 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.623},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.63100000000000001}]},{"osd":5,"last update":"Tue Apr 2 14:52:53 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.56599999999999995}]},{"osd":6,"last update":"Tue Apr 2 14:53:19 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.60699999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66500000000000004}]}]},{"osd":3,"up_from":27,"seq":115964117025,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 14:53:08 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39300000000000002,"15min":0.39200000000000002},"min":{"1min":0.28000000000000003,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.40200000000000002}]},{"osd":1,"last update":"Tue Apr 2 14:53:08 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.41899999999999998,"15min":0.41699999999999998},"min":{"1min":0.29299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.59299999999999997,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.44500000000000001}]},{"osd":2,"last update":"Tue Apr 2 14:53:08 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.437,"15min":0.439},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.501},{"interface":"front","average":{"1min":0.434,"5min":0.433,"15min":0.433},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.436}]},{"osd":4,"last update":"Tue Apr 2 14:52:34 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.83099999999999996},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.69399999999999995}]},{"osd":5,"last update":"Tue Apr 2 14:52:53 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.80800000000000005}]},{"osd":6,"last update":"Tue Apr 2 14:53:17 2024","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":4.4710000000000001,"5min":4.4710000000000001,"15min":4.4710000000000001},"last":0.88100000000000001},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.72399999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73499999999999999}]}]},{"osd":4,"up_from":32,"seq":137438953500,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 14:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.51600000000000001,"15min":0.504},"min":{"1min":0.39200000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.55300000000000005,"15min":0.55100000000000005},"min":{"1min":0.41499999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.47999999999999998}]},{"osd":1,"last update":"Tue Apr 2 14:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.53700000000000003,"15min":0.53400000000000003},"min":{"1min":0.45100000000000001,"5min":0.374,"15min":0.374},"max":{"1min":0.69699999999999995,"5min":0.755,"15min":0.755},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.53200000000000003,"15min":0.52600000000000002},"min":{"1min":0.40100000000000002,"5min":0.371,"15min":0.371},"max":{"1min":0.76900000000000002,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.441}]},{"osd":2,"last update":"Tue Apr 2 14:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.40699999999999997,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.68799999999999994,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.53600000000000003,"15min":0.53200000000000003},"min":{"1min":0.42699999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.70199999999999996,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.52100000000000002}]},{"osd":3,"last update":"Tue Apr 2 14:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.53600000000000003,"15min":0.53200000000000003},"min":{"1min":0.41699999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.54200000000000004,"15min":0.53200000000000003},"min":{"1min":0.52100000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.70199999999999996,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.59399999999999997}]},{"osd":5,"last update":"Tue Apr 2 14:52:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.47099999999999997}]},{"osd":6,"last update":"Tue Apr 2 14:53:19 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.505}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54800000000000004}]}]},{"osd":5,"up_from":40,"seq":171798691863,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5664,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739488,"statfs":{"total":95995035648,"available":95989235712,"internally_reserved":0,"allocated":614400,"data_stored":247949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 14:52:54 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.501}]},{"osd":1,"last update":"Tue Apr 2 14:52:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.46899999999999997}]},{"osd":2,"last update":"Tue Apr 2 14:52:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.44}]},{"osd":3,"last update":"Tue Apr 2 14:52:54 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.52400000000000002}]},{"osd":4,"last update":"Tue Apr 2 14:52:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.44900000000000001}]},{"osd":6,"last update":"Tue Apr 2 14:53:19 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.58699999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41899999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-02T14:53:37.651 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-02T14:53:37.864 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:38.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:38 smithi122 ceph-mon[24801]: from='client.14438 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T14:53:38.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:38 smithi122 ceph-mon[24801]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:38.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:38 smithi183 ceph-mon[27058]: from='client.14438 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T14:53:38.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:38 smithi183 ceph-mon[27058]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:39.506 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:53:39.508 INFO:teuthology.orchestra.run.smithi122.stderr:dumped all 2024-04-02T14:53:40.155 INFO:teuthology.orchestra.run.smithi122.stdout:{"pg_ready":true,"pg_map":{"version":203,"stamp":"2024-04-02T14:53:39.222443+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48808,"kb_used_data":6376,"kb_used_omap":0,"kb_used_meta":42368,"kb_avail":749912408,"statfs":{"total":767960285184,"available":767910305792,"internally_reserved":0,"allocated":6529024,"data_stored":3571378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43384832},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001637"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":222,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-04-02T14:52:44.576111+0000","last_change":"2024-04-02T14:52:44.576111+0000","last_active":"2024-04-02T14:52:44.576111+0000","last_peered":"2024-04-02T14:52:44.576111+0000","last_clean":"2024-04-02T14:52:44.576111+0000","last_became_active":"2024-04-02T14:52:44.575660+0000","last_became_peered":"2024-04-02T14:52:44.575660+0000","last_unstale":"2024-04-02T14:52:44.576111+0000","last_undegraded":"2024-04-02T14:52:44.576111+0000","last_fullsized":"2024-04-02T14:52:44.576111+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-02T14:50:41.534364+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-02T14:50:41.534364+0000","last_clean_scrub_stamp":"2024-04-02T14:50:41.534364+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-04T00:54:34.035198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168590,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776819,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64300000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56100000000000005}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57699999999999996}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70399999999999996}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.77500000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73799999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72699999999999998}]}]},{"osd":6,"up_from":48,"seq":206158430227,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6184,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738968,"statfs":{"total":95995035648,"available":95988703232,"internally_reserved":0,"allocated":1146880,"data_stored":776877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 14:53:17 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.38900000000000001}]},{"osd":1,"last update":"Tue Apr 2 14:53:17 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.45800000000000002}]},{"osd":2,"last update":"Tue Apr 2 14:53:17 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.505}]},{"osd":3,"last update":"Tue Apr 2 14:53:17 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.55300000000000005}]},{"osd":4,"last update":"Tue Apr 2 14:53:17 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.441},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.48899999999999999}]},{"osd":5,"last update":"Tue Apr 2 14:53:17 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.52700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47599999999999998}]}]},{"osd":1,"up_from":14,"seq":60129542189,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 14:53:17 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.34499999999999997,"15min":0.32300000000000001},"min":{"1min":0.26400000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":1.3380000000000001,"5min":1.3380000000000001,"15min":1.3380000000000001},"last":0.372},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.35599999999999998,"15min":0.33000000000000002},"min":{"1min":0.252,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.40799999999999997}]},{"osd":2,"last update":"Tue Apr 2 14:52:43 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.38400000000000001,"15min":0.378},"min":{"1min":0.27100000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.442,"5min":0.40500000000000003,"15min":0.39900000000000002},"min":{"1min":0.32900000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.39700000000000002}]},{"osd":3,"last update":"Tue Apr 2 14:53:11 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.435,"15min":0.44},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.443},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.38200000000000001}]},{"osd":4,"last update":"Tue Apr 2 14:53:34 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.55900000000000005,"15min":0.55000000000000004},"min":{"1min":0.47499999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.997,"5min":0.997,"15min":0.997},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.60299999999999998,"15min":0.59399999999999997},"min":{"1min":0.48799999999999999,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.083,"5min":1.083,"15min":1.083},"last":0.52500000000000002}]},{"osd":5,"last update":"Tue Apr 2 14:52:56 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.54500000000000004}]},{"osd":6,"last update":"Tue Apr 2 14:53:17 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.55900000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50900000000000001}]}]},{"osd":0,"up_from":9,"seq":38654705716,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Apr 2 14:53:18 2024","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.34399999999999997,"15min":0.33200000000000002},"min":{"1min":0.23000000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.52500000000000002,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.36099999999999999,"5min":0.34999999999999998,"15min":0.34000000000000002},"min":{"1min":0.248,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.55400000000000005}]},{"osd":2,"last update":"Tue Apr 2 14:52:45 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.40500000000000003,"15min":0.40100000000000002},"min":{"1min":0.29499999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.51200000000000001,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.28799999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.54100000000000004,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.53100000000000003}]},{"osd":3,"last update":"Tue Apr 2 14:53:11 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.44,"15min":0.44800000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.55300000000000005,"5min":0.622,"15min":0.622},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.372,"5min":0.42599999999999999,"15min":0.435},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.503,"5min":0.626,"15min":0.626},"last":0.497}]},{"osd":4,"last update":"Tue Apr 2 14:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58899999999999997,"15min":0.58999999999999997},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.82699999999999996,"5min":0.997,"15min":0.997},"last":0.84899999999999998},{"interface":"front","average":{"1min":0.627,"5min":0.60799999999999998,"15min":0.60499999999999998},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.78700000000000003,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.82999999999999996}]},{"osd":5,"last update":"Tue Apr 2 14:52:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.80500000000000005}]},{"osd":6,"last update":"Tue Apr 2 14:53:18 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.75900000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73299999999999998}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6240,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738912,"statfs":{"total":95995035648,"available":95988645888,"internally_reserved":0,"allocated":614400,"data_stored":247949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 14:52:43 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.379,"15min":0.378},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.378,"5min":0.36599999999999999,"15min":0.36399999999999999},"min":{"1min":0.27700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.33700000000000002}]},{"osd":1,"last update":"Tue Apr 2 14:52:43 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.39200000000000002,"15min":0.39500000000000002},"min":{"1min":0.26300000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.41099999999999998,"15min":0.40799999999999997},"min":{"1min":0.29999999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.54600000000000004,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.36499999999999999}]},{"osd":3,"last update":"Tue Apr 2 14:53:10 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.437,"15min":0.441},"min":{"1min":0.318,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.432,"5min":0.45100000000000001,"15min":0.45400000000000001},"min":{"1min":0.311,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.317}]},{"osd":4,"last update":"Tue Apr 2 14:53:32 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.63,"5min":0.59799999999999998,"15min":0.59199999999999997},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.51000000000000001}]},{"osd":5,"last update":"Tue Apr 2 14:52:53 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.46999999999999997}]},{"osd":6,"last update":"Tue Apr 2 14:53:19 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.44400000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53300000000000003}]}]},{"osd":3,"up_from":27,"seq":115964117026,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 14:53:08 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39300000000000002,"15min":0.39200000000000002},"min":{"1min":0.28000000000000003,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.435}]},{"osd":1,"last update":"Tue Apr 2 14:53:08 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.41899999999999998,"15min":0.41699999999999998},"min":{"1min":0.29299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.59299999999999997,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.46899999999999997}]},{"osd":2,"last update":"Tue Apr 2 14:53:08 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.437,"15min":0.439},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.434,"5min":0.433,"15min":0.433},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.46000000000000002}]},{"osd":4,"last update":"Tue Apr 2 14:53:34 2024","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.623,"15min":0.60299999999999998},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":4.46,"5min":4.46,"15min":4.46},"last":0.753},{"interface":"front","average":{"1min":0.75,"5min":0.61399999999999999,"15min":0.59099999999999997},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":4.508,"5min":4.508,"15min":4.508},"last":0.66300000000000003}]},{"osd":5,"last update":"Tue Apr 2 14:52:53 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.64100000000000001}]},{"osd":6,"last update":"Tue Apr 2 14:53:17 2024","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":4.4710000000000001,"5min":4.4710000000000001,"15min":4.4710000000000001},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.77700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.622}]}]},{"osd":4,"up_from":32,"seq":137438953500,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 14:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.51600000000000001,"15min":0.504},"min":{"1min":0.39200000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.55300000000000005,"15min":0.55100000000000005},"min":{"1min":0.41499999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.47999999999999998}]},{"osd":1,"last update":"Tue Apr 2 14:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.53700000000000003,"15min":0.53400000000000003},"min":{"1min":0.45100000000000001,"5min":0.374,"15min":0.374},"max":{"1min":0.69699999999999995,"5min":0.755,"15min":0.755},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.53200000000000003,"15min":0.52600000000000002},"min":{"1min":0.40100000000000002,"5min":0.371,"15min":0.371},"max":{"1min":0.76900000000000002,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.441}]},{"osd":2,"last update":"Tue Apr 2 14:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.40699999999999997,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.68799999999999994,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.53600000000000003,"15min":0.53200000000000003},"min":{"1min":0.42699999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.70199999999999996,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.52100000000000002}]},{"osd":3,"last update":"Tue Apr 2 14:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.53600000000000003,"15min":0.53200000000000003},"min":{"1min":0.41699999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.54200000000000004,"15min":0.53200000000000003},"min":{"1min":0.52100000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.70199999999999996,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.59399999999999997}]},{"osd":5,"last update":"Tue Apr 2 14:52:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.47099999999999997}]},{"osd":6,"last update":"Tue Apr 2 14:53:19 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.505}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54800000000000004}]}]},{"osd":5,"up_from":40,"seq":171798691864,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5664,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739488,"statfs":{"total":95995035648,"available":95989235712,"internally_reserved":0,"allocated":614400,"data_stored":247949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 14:52:54 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.58699999999999997}]},{"osd":1,"last update":"Tue Apr 2 14:52:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.503}]},{"osd":2,"last update":"Tue Apr 2 14:52:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.42899999999999999}]},{"osd":3,"last update":"Tue Apr 2 14:52:54 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.60299999999999998}]},{"osd":4,"last update":"Tue Apr 2 14:52:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.48799999999999999}]},{"osd":6,"last update":"Tue Apr 2 14:53:19 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.40500000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35099999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-02T14:53:40.156 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-02T14:53:40.156 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-02T14:53:40.157 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-02T14:53:40.157 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph health --format=json 2024-04-02T14:53:40.376 INFO:teuthology.orchestra.run.smithi122.stderr:Inferring config /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/mon.smithi122/config 2024-04-02T14:53:40.499 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:40 smithi122 ceph-mon[24801]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:40.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:40 smithi183 ceph-mon[27058]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:41.497 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:41 smithi122 ceph-mon[24801]: from='client.14442 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T14:53:41.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:41 smithi183 ceph-mon[27058]: from='client.14442 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T14:53:42.157 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:53:42.157 INFO:teuthology.orchestra.run.smithi122.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-02T14:53:42.488 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:42 smithi122 ceph-mon[24801]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:42.488 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:42 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/136801941' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-02T14:53:42.528 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:42 smithi183 ceph-mon[27058]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:42.528 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:42 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/136801941' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-02T14:53:42.847 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-02T14:53:42.847 INFO:tasks.cephadm:Setup complete, yielding 2024-04-02T14:53:42.847 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-02T14:53:42.857 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi122.front.sepia.ceph.com 2024-04-02T14:53:42.857 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- bash -c 'ceph orch status' 2024-04-02T14:53:43.424 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:53:43.424 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:53:43.424 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:53:43.424 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:43 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:53:43.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:53:43.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:53:43.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:53:43.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:43 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:53:44.485 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:44 smithi122 ceph-mon[24801]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:44.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:44 smithi183 ceph-mon[27058]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:44.855 INFO:teuthology.orchestra.run.smithi122.stdout:Backend: cephadm 2024-04-02T14:53:44.856 INFO:teuthology.orchestra.run.smithi122.stdout:Available: Yes 2024-04-02T14:53:44.856 INFO:teuthology.orchestra.run.smithi122.stdout:Paused: No 2024-04-02T14:53:45.492 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- bash -c 'ceph orch ps' 2024-04-02T14:53:45.799 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:45 smithi122 ceph-mon[24801]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:53:45.799 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:45 smithi122 ceph-mon[24801]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:45.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:45 smithi183 ceph-mon[27058]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:53:45.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:45 smithi183 ceph-mon[27058]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:47.353 INFO:teuthology.orchestra.run.smithi122.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T14:53:47.353 INFO:teuthology.orchestra.run.smithi122.stdout:alertmanager.smithi122 smithi122 *:9093,9094 running (4m) 2m ago 5m 17.6M - ba2b418f427c 87ea6de784f7 2024-04-02T14:53:47.354 INFO:teuthology.orchestra.run.smithi122.stdout:crash.smithi122 smithi122 running (5m) 2m ago 5m 6400k - 17.2.0 e1d6a67b021e 140dc3b57d86 2024-04-02T14:53:47.354 INFO:teuthology.orchestra.run.smithi122.stdout:crash.smithi183 smithi183 running (5m) 67s ago 5m 6400k - 17.2.0 e1d6a67b021e 1262330a9cca 2024-04-02T14:53:47.354 INFO:teuthology.orchestra.run.smithi122.stdout:grafana.smithi122 smithi122 *:3000 running (4m) 2m ago 5m 44.9M - 8.3.5 dad864ee21e9 f0623c4719c2 2024-04-02T14:53:47.354 INFO:teuthology.orchestra.run.smithi122.stdout:mgr.smithi122.szjkfg smithi122 *:9283 running (7m) 2m ago 7m 447M - 17.2.0 e1d6a67b021e 7583b48bda3e 2024-04-02T14:53:47.354 INFO:teuthology.orchestra.run.smithi122.stdout:mgr.smithi183.smxedq smithi183 *:8443,9283 running (5m) 67s ago 5m 413M - 17.2.0 e1d6a67b021e c066bca1a154 2024-04-02T14:53:47.354 INFO:teuthology.orchestra.run.smithi122.stdout:mon.smithi122 smithi122 running (7m) 2m ago 7m 42.1M 2048M 17.2.0 e1d6a67b021e d8f91f766e22 2024-04-02T14:53:47.354 INFO:teuthology.orchestra.run.smithi122.stdout:mon.smithi183 smithi183 running (4m) 67s ago 4m 40.0M 2048M 17.2.0 e1d6a67b021e d6d4734135ac 2024-04-02T14:53:47.354 INFO:teuthology.orchestra.run.smithi122.stdout:node-exporter.smithi122 smithi122 *:9100 running (5m) 2m ago 5m 18.2M - 1dbe0e931976 8815d797e9e2 2024-04-02T14:53:47.354 INFO:teuthology.orchestra.run.smithi122.stdout:node-exporter.smithi183 smithi183 *:9100 running (4m) 67s ago 4m 17.2M - 1dbe0e931976 6c7613782142 2024-04-02T14:53:47.354 INFO:teuthology.orchestra.run.smithi122.stdout:osd.0 smithi122 running (4m) 2m ago 4m 32.3M 3216M 17.2.0 e1d6a67b021e bee30cdc6385 2024-04-02T14:53:47.354 INFO:teuthology.orchestra.run.smithi122.stdout:osd.1 smithi122 running (3m) 2m ago 3m 34.1M 3216M 17.2.0 e1d6a67b021e 92d52898245e 2024-04-02T14:53:47.354 INFO:teuthology.orchestra.run.smithi122.stdout:osd.2 smithi122 running (3m) 2m ago 3m 31.2M 3216M 17.2.0 e1d6a67b021e b88321d239b1 2024-04-02T14:53:47.354 INFO:teuthology.orchestra.run.smithi122.stdout:osd.3 smithi122 running (2m) 2m ago 2m 13.4M 3216M 17.2.0 e1d6a67b021e 1aec61f33b52 2024-04-02T14:53:47.354 INFO:teuthology.orchestra.run.smithi122.stdout:osd.4 smithi183 running (2m) 67s ago 2m 33.3M 3984M 17.2.0 e1d6a67b021e 0d1dbe82a342 2024-04-02T14:53:47.354 INFO:teuthology.orchestra.run.smithi122.stdout:osd.5 smithi183 running (2m) 67s ago 2m 35.7M 3984M 17.2.0 e1d6a67b021e 9bc6c62fa5cb 2024-04-02T14:53:47.355 INFO:teuthology.orchestra.run.smithi122.stdout:osd.6 smithi183 running (97s) 67s ago 97s 30.8M 3984M 17.2.0 e1d6a67b021e f9c8a0a7bfdc 2024-04-02T14:53:47.355 INFO:teuthology.orchestra.run.smithi122.stdout:osd.7 smithi183 running (73s) 67s ago 72s 13.3M 3984M 17.2.0 e1d6a67b021e 1406cf2e45a8 2024-04-02T14:53:47.355 INFO:teuthology.orchestra.run.smithi122.stdout:prometheus.smithi122 smithi122 *:9095 running (4m) 2m ago 4m 53.9M - 514e6a882f6e be604b0d0ed3 2024-04-02T14:53:47.978 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- bash -c 'ceph orch ls' 2024-04-02T14:53:48.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:48 smithi122 ceph-mon[24801]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:48.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:48 smithi122 ceph-mon[24801]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:53:48.602 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:48 smithi183 ceph-mon[27058]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:48.602 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:48 smithi183 ceph-mon[27058]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:53:49.868 INFO:teuthology.orchestra.run.smithi122.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-02T14:53:49.869 INFO:teuthology.orchestra.run.smithi122.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-04-02T14:53:49.869 INFO:teuthology.orchestra.run.smithi122.stdout:crash 2/2 2m ago 6m * 2024-04-02T14:53:49.869 INFO:teuthology.orchestra.run.smithi122.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-04-02T14:53:49.869 INFO:teuthology.orchestra.run.smithi122.stdout:mgr 2/2 2m ago 6m count:2 2024-04-02T14:53:49.869 INFO:teuthology.orchestra.run.smithi122.stdout:mon 2/2 2m ago 5m smithi122:172.21.15.122=smithi122;smithi183:172.21.15.183=smithi183;count:2 2024-04-02T14:53:49.869 INFO:teuthology.orchestra.run.smithi122.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-04-02T14:53:49.869 INFO:teuthology.orchestra.run.smithi122.stdout:osd 8 2m ago - 2024-04-02T14:53:49.869 INFO:teuthology.orchestra.run.smithi122.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-04-02T14:53:50.543 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:50 smithi122 ceph-mon[24801]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:50.568 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- bash -c 'ceph orch host ls' 2024-04-02T14:53:50.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:50 smithi183 ceph-mon[27058]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:51.549 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:51 smithi122 ceph-mon[24801]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:53:51.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:51 smithi183 ceph-mon[27058]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:53:52.544 INFO:teuthology.orchestra.run.smithi122.stdout:HOST ADDR LABELS STATUS 2024-04-02T14:53:52.544 INFO:teuthology.orchestra.run.smithi122.stdout:smithi122 172.21.15.122 2024-04-02T14:53:52.544 INFO:teuthology.orchestra.run.smithi122.stdout:smithi183 172.21.15.183 2024-04-02T14:53:52.545 INFO:teuthology.orchestra.run.smithi122.stdout:2 hosts in cluster 2024-04-02T14:53:52.545 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:52 smithi122 ceph-mon[24801]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:52.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:52 smithi183 ceph-mon[27058]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:53.147 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- bash -c 'ceph orch device ls' 2024-04-02T14:53:54.530 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:54 smithi122 ceph-mon[24801]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:53:54.530 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:54 smithi122 ceph-mon[24801]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:54.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:54 smithi183 ceph-mon[27058]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:53:54.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:54 smithi183 ceph-mon[27058]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:55.020 INFO:teuthology.orchestra.run.smithi122.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-04-02T14:53:55.020 INFO:teuthology.orchestra.run.smithi122.stdout:smithi122 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT6204010U400BGN 400G LVM detected, locked 2024-04-02T14:53:55.020 INFO:teuthology.orchestra.run.smithi122.stdout:smithi183 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620500QJ400BGN 400G LVM detected, locked 2024-04-02T14:53:55.635 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-02T14:53:55.649 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi122.front.sepia.ceph.com 2024-04-02T14:53:55.649 DEBUG:teuthology.orchestra.run.smithi122:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-04-02T14:53:55.677 INFO:teuthology.orchestra.run.smithi122.stderr:+ systemctl stop nfs-server 2024-04-02T14:53:55.683 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi183.front.sepia.ceph.com 2024-04-02T14:53:55.684 DEBUG:teuthology.orchestra.run.smithi183:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-04-02T14:53:55.715 INFO:teuthology.orchestra.run.smithi183.stderr:+ systemctl stop nfs-server 2024-04-02T14:53:55.723 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-02T14:53:55.735 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi122.front.sepia.ceph.com 2024-04-02T14:53:55.735 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- bash -c 'ceph fs volume create foofs' 2024-04-02T14:53:55.758 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:55 smithi122 ceph-mon[24801]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:53:55.758 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:55 smithi122 ceph-mon[24801]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:55.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:55 smithi183 ceph-mon[27058]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:53:55.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:55 smithi183 ceph-mon[27058]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:58.838 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:58 smithi122 ceph-mon[24801]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:58.838 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:58 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-04-02T14:53:58.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:58 smithi183 ceph-mon[27058]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:58.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:58 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-04-02T14:53:59.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi122[24797]: 2024-04-02T14:53:59.285+0000 7f80057c7700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-02T14:53:59.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:53:59.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-04-02T14:53:59.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: osdmap e61: 8 total, 8 up, 8 in 2024-04-02T14:53:59.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-04-02T14:53:59.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: pgmap v214: 33 pgs: 2 creating+peering, 1 active+clean, 30 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:59.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-04-02T14:53:59.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: osdmap e62: 8 total, 8 up, 8 in 2024-04-02T14:53:59.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-04-02T14:53:59.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-02T14:53:59.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-02T14:53:59.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: osdmap e63: 8 total, 8 up, 8 in 2024-04-02T14:53:59.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-04-02T14:53:59.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: fsmap foofs:0 2024-04-02T14:53:59.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: Saving service mds.foofs spec with placement count:2 2024-04-02T14:53:59.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:53:59.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:53:59.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:53:59.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:53:59.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:53:59.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:53:59.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi122.cghwpx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-02T14:53:59.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi122.cghwpx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-02T14:53:59.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:53:59.590 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:53:59 smithi122 ceph-mon[24801]: Deploying daemon mds.foofs.smithi122.cghwpx on smithi122 2024-04-02T14:53:59.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:53:59.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-04-02T14:53:59.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: osdmap e61: 8 total, 8 up, 8 in 2024-04-02T14:53:59.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-04-02T14:53:59.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: pgmap v214: 33 pgs: 2 creating+peering, 1 active+clean, 30 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:53:59.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-04-02T14:53:59.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: osdmap e62: 8 total, 8 up, 8 in 2024-04-02T14:53:59.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-04-02T14:53:59.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-02T14:53:59.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-02T14:53:59.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: osdmap e63: 8 total, 8 up, 8 in 2024-04-02T14:53:59.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-04-02T14:53:59.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: fsmap foofs:0 2024-04-02T14:53:59.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: Saving service mds.foofs spec with placement count:2 2024-04-02T14:53:59.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:53:59.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:53:59.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:53:59.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:53:59.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:53:59.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:53:59.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi122.cghwpx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-02T14:53:59.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi122.cghwpx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-02T14:53:59.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:53:59.901 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:53:59 smithi183 ceph-mon[27058]: Deploying daemon mds.foofs.smithi122.cghwpx on smithi122 2024-04-02T14:54:00.131 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-04-02T14:54:00.140 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-04-02T14:54:00.140 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph orch ls -f json 2024-04-02T14:54:01.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:01 smithi122 ceph-mon[24801]: osdmap e64: 8 total, 8 up, 8 in 2024-04-02T14:54:01.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:01 smithi183 ceph-mon[27058]: osdmap e64: 8 total, 8 up, 8 in 2024-04-02T14:54:02.537 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:02 smithi122 ceph-mon[24801]: pgmap v218: 65 pgs: 6 creating+peering, 18 active+clean, 41 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:54:02.537 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:02 smithi122 ceph-mon[24801]: osdmap e65: 8 total, 8 up, 8 in 2024-04-02T14:54:02.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:02 smithi183 ceph-mon[27058]: pgmap v218: 65 pgs: 6 creating+peering, 18 active+clean, 41 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:54:02.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:02 smithi183 ceph-mon[27058]: osdmap e65: 8 total, 8 up, 8 in 2024-04-02T14:54:03.213 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:54:03.213 INFO:teuthology.orchestra.run.smithi122.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-02T14:47:26.282426Z", "last_refresh": "2024-04-02T14:51:06.808137Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-02T14:47:20.073967Z", "last_refresh": "2024-04-02T14:51:06.808199Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-02T14:47:23.621258Z", "last_refresh": "2024-04-02T14:51:06.808259Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-02T14:53:59.303575Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-02T14:53:59.298466Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-02T14:47:18.018869Z", "last_refresh": "2024-04-02T14:51:06.808050Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi122:172.21.15.122=smithi122", "smithi183:172.21.15.183=smithi183"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-02T14:48:08.043872Z", "last_refresh": "2024-04-02T14:51:06.807909Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-02T14:47:25.009565Z", "last_refresh": "2024-04-02T14:51:06.808319Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-02T14:51:06.808437Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-02T14:47:21.909680Z", "last_refresh": "2024-04-02T14:51:06.808378Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-02T14:54:04.196 INFO:tasks.cephadm:mds.foofs has 0/2 2024-04-02T14:54:04.531 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:04 smithi183 ceph-mon[27058]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T14:54:04.531 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:04 smithi183 ceph-mon[27058]: pgmap v220: 65 pgs: 6 creating+peering, 53 active+clean, 6 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:54:04.531 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:04 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:04.531 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:04 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi183.idnham", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-02T14:54:04.531 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:04 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi183.idnham", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-02T14:54:04.531 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:04 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:04.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:04 smithi122 ceph-mon[24801]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T14:54:04.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:04 smithi122 ceph-mon[24801]: pgmap v220: 65 pgs: 6 creating+peering, 53 active+clean, 6 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:54:04.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:04 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:04.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:04 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi183.idnham", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-02T14:54:04.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:04 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi183.idnham", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-02T14:54:04.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:04 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:05.197 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph orch ls -f json 2024-04-02T14:54:05.523 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:05 smithi122 ceph-mon[24801]: Deploying daemon mds.foofs.smithi183.idnham on smithi183 2024-04-02T14:54:05.523 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:05 smithi122 ceph-mon[24801]: mds.? [v2:172.21.15.122:6834/2012625988,v1:172.21.15.122:6835/2012625988] up:boot 2024-04-02T14:54:05.524 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:05 smithi122 ceph-mon[24801]: daemon mds.foofs.smithi122.cghwpx assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-04-02T14:54:05.524 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:05 smithi122 ceph-mon[24801]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-02T14:54:05.524 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:05 smithi122 ceph-mon[24801]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-02T14:54:05.524 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:05 smithi122 ceph-mon[24801]: Cluster is now healthy 2024-04-02T14:54:05.524 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:05 smithi122 ceph-mon[24801]: fsmap foofs:0 1 up:standby 2024-04-02T14:54:05.524 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:05 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi122.cghwpx"}]: dispatch 2024-04-02T14:54:05.524 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:05 smithi122 ceph-mon[24801]: fsmap foofs:1 {0=foofs.smithi122.cghwpx=up:creating} 2024-04-02T14:54:05.524 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:05 smithi122 ceph-mon[24801]: daemon mds.foofs.smithi122.cghwpx is now active in filesystem foofs as rank 0 2024-04-02T14:54:05.635 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:05 smithi183 ceph-mon[27058]: Deploying daemon mds.foofs.smithi183.idnham on smithi183 2024-04-02T14:54:05.635 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:05 smithi183 ceph-mon[27058]: mds.? [v2:172.21.15.122:6834/2012625988,v1:172.21.15.122:6835/2012625988] up:boot 2024-04-02T14:54:05.635 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:05 smithi183 ceph-mon[27058]: daemon mds.foofs.smithi122.cghwpx assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-04-02T14:54:05.635 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:05 smithi183 ceph-mon[27058]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-02T14:54:05.635 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:05 smithi183 ceph-mon[27058]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-02T14:54:05.635 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:05 smithi183 ceph-mon[27058]: Cluster is now healthy 2024-04-02T14:54:05.635 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:05 smithi183 ceph-mon[27058]: fsmap foofs:0 1 up:standby 2024-04-02T14:54:05.635 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:05 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi122.cghwpx"}]: dispatch 2024-04-02T14:54:05.635 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:05 smithi183 ceph-mon[27058]: fsmap foofs:1 {0=foofs.smithi122.cghwpx=up:creating} 2024-04-02T14:54:05.635 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:05 smithi183 ceph-mon[27058]: daemon mds.foofs.smithi122.cghwpx is now active in filesystem foofs as rank 0 2024-04-02T14:54:06.551 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:06 smithi122 ceph-mon[24801]: pgmap v221: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 172 B/s wr, 0 op/s 2024-04-02T14:54:06.551 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:06 smithi122 ceph-mon[24801]: mds.? [v2:172.21.15.122:6834/2012625988,v1:172.21.15.122:6835/2012625988] up:active 2024-04-02T14:54:06.551 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:06 smithi122 ceph-mon[24801]: fsmap foofs:1 {0=foofs.smithi122.cghwpx=up:active} 2024-04-02T14:54:06.551 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:06 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:06.551 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:06 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:54:06.551 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:06 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:06.551 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:06 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:54:06.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:06 smithi183 ceph-mon[27058]: pgmap v221: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 172 B/s wr, 0 op/s 2024-04-02T14:54:06.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:06 smithi183 ceph-mon[27058]: mds.? [v2:172.21.15.122:6834/2012625988,v1:172.21.15.122:6835/2012625988] up:active 2024-04-02T14:54:06.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:06 smithi183 ceph-mon[27058]: fsmap foofs:1 {0=foofs.smithi122.cghwpx=up:active} 2024-04-02T14:54:06.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:06 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:06.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:06 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:54:06.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:06 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:06.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:06 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:54:07.278 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:54:07.279 INFO:teuthology.orchestra.run.smithi122.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-02T14:47:26.282426Z", "last_refresh": "2024-04-02T14:51:06.808137Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-02T14:47:20.073967Z", "last_refresh": "2024-04-02T14:51:06.808199Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-02T14:47:23.621258Z", "last_refresh": "2024-04-02T14:51:06.808259Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-02T14:53:59.303575Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-02T14:53:59.298466Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-02T14:47:18.018869Z", "last_refresh": "2024-04-02T14:51:06.808050Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi122:172.21.15.122=smithi122", "smithi183:172.21.15.183=smithi183"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-02T14:48:08.043872Z", "last_refresh": "2024-04-02T14:51:06.807909Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-02T14:47:25.009565Z", "last_refresh": "2024-04-02T14:51:06.808319Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-02T14:51:06.808437Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-02T14:47:21.909680Z", "last_refresh": "2024-04-02T14:51:06.808378Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-02T14:54:07.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:07 smithi183 ceph-mon[27058]: mds.? [v2:172.21.15.183:6832/1607898514,v1:172.21.15.183:6833/1607898514] up:boot 2024-04-02T14:54:07.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:07 smithi183 ceph-mon[27058]: fsmap foofs:1 {0=foofs.smithi122.cghwpx=up:active} 1 up:standby 2024-04-02T14:54:07.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:07 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi183.idnham"}]: dispatch 2024-04-02T14:54:07.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:07 smithi183 ceph-mon[27058]: fsmap foofs:1 {0=foofs.smithi122.cghwpx=up:active} 1 up:standby 2024-04-02T14:54:07.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:07 smithi183 ceph-mon[27058]: pgmap v222: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 129 B/s wr, 2 op/s 2024-04-02T14:54:07.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:07 smithi183 ceph-mon[27058]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T14:54:07.838 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:07 smithi122 ceph-mon[24801]: mds.? [v2:172.21.15.183:6832/1607898514,v1:172.21.15.183:6833/1607898514] up:boot 2024-04-02T14:54:07.838 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:07 smithi122 ceph-mon[24801]: fsmap foofs:1 {0=foofs.smithi122.cghwpx=up:active} 1 up:standby 2024-04-02T14:54:07.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:07 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi183.idnham"}]: dispatch 2024-04-02T14:54:07.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:07 smithi122 ceph-mon[24801]: fsmap foofs:1 {0=foofs.smithi122.cghwpx=up:active} 1 up:standby 2024-04-02T14:54:07.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:07 smithi122 ceph-mon[24801]: pgmap v222: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 129 B/s wr, 2 op/s 2024-04-02T14:54:07.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:07 smithi122 ceph-mon[24801]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T14:54:08.163 INFO:tasks.cephadm:mds.foofs has 0/2 2024-04-02T14:54:09.164 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- ceph orch ls -f json 2024-04-02T14:54:09.529 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:09 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:09.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:09 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:10.382 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:10 smithi122 ceph-mon[24801]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-04-02T14:54:10.382 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:10 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:10.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:10 smithi183 ceph-mon[27058]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-04-02T14:54:10.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:10 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:12.338 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:12 smithi122 ceph-mon[24801]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-02T14:54:12.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:12 smithi183 ceph-mon[27058]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-02T14:54:13.159 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:54:13.159 INFO:teuthology.orchestra.run.smithi122.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-02T14:47:26.282426Z", "last_refresh": "2024-04-02T14:54:13.148629Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-02T14:47:20.073967Z", "last_refresh": "2024-04-02T14:54:08.231426Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-02T14:47:23.621258Z", "last_refresh": "2024-04-02T14:54:13.148763Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-02T14:53:59.303575Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-02T14:53:59.298466Z", "last_refresh": "2024-04-02T14:54:08.232047Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-02T14:47:18.018869Z", "last_refresh": "2024-04-02T14:54:08.231575Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi122:172.21.15.122=smithi122", "smithi183:172.21.15.183=smithi183"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-02T14:48:08.043872Z", "last_refresh": "2024-04-02T14:54:08.231649Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-02T14:47:25.009565Z", "last_refresh": "2024-04-02T14:54:08.231714Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-02T14:54:08.231779Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-02T14:47:21.909680Z", "last_refresh": "2024-04-02T14:54:13.148896Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-02T14:54:13.798 INFO:tasks.cephadm:mds.foofs has 2/2 2024-04-02T14:54:13.798 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-02T14:54:13.807 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi122.front.sepia.ceph.com 2024-04-02T14:54:13.807 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-04-02T14:54:14.338 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:14 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:14.461 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:14 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:14.461 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:14 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:54:14.461 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:14 smithi122 ceph-mon[24801]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T14:54:14.461 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:14 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:14.462 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:14 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:54:14.462 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:14 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:14.462 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:14 smithi122 ceph-mon[24801]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-02T14:54:14.462 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:14 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:14.462 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:14 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:14.462 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:14 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:54:14.463 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:14 smithi183 ceph-mon[27058]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T14:54:14.463 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:14 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:14.463 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:14 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:54:14.463 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:14 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:14.463 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:14 smithi183 ceph-mon[27058]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-02T14:54:15.452 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:15 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:15.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:15 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:16.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:16 smithi122 ceph-mon[24801]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-02T14:54:16.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:16 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-04-02T14:54:16.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:16 smithi183 ceph-mon[27058]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-02T14:54:16.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:16 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-04-02T14:54:17.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:17 smithi122 ceph-mon[24801]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:17.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:17 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-04-02T14:54:17.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:17 smithi122 ceph-mon[24801]: osdmap e66: 8 total, 8 up, 8 in 2024-04-02T14:54:17.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:17 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-02T14:54:17.612 INFO:teuthology.orchestra.run.smithi122.stdout:NFS Cluster Created Successfully 2024-04-02T14:54:17.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:17 smithi183 ceph-mon[27058]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:17.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:17 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-04-02T14:54:17.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:17 smithi183 ceph-mon[27058]: osdmap e66: 8 total, 8 up, 8 in 2024-04-02T14:54:17.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:17 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-02T14:54:18.320 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-04-02T14:54:18.515 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:18 smithi122 ceph-mon[24801]: pgmap v228: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 2 op/s 2024-04-02T14:54:18.516 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:18 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-04-02T14:54:18.516 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:18 smithi122 ceph-mon[24801]: osdmap e67: 8 total, 8 up, 8 in 2024-04-02T14:54:18.516 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:18 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:18.516 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:18 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:54:18.516 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:18 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:18.516 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:18 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:54:18.516 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:18 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:18.516 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:18 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:18.516 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:18 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi122.iqmcyp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-02T14:54:18.516 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:18 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi122.iqmcyp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-02T14:54:18.516 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:18 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-02T14:54:18.516 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:18 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-02T14:54:18.516 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:18 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:18.516 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:18 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-02T14:54:18.516 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:18 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-02T14:54:18.517 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:18 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi122.iqmcyp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T14:54:18.517 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:18 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi122.iqmcyp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T14:54:18.517 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:18 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:18.573 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:18 smithi183 ceph-mon[27058]: pgmap v228: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 2 op/s 2024-04-02T14:54:18.573 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:18 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-04-02T14:54:18.573 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:18 smithi183 ceph-mon[27058]: osdmap e67: 8 total, 8 up, 8 in 2024-04-02T14:54:18.573 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:18 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:18.573 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:18 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:54:18.573 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:18 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:18.573 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:18 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:54:18.573 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:18 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:18.574 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:18 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:18.574 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:18 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi122.iqmcyp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-02T14:54:18.574 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:18 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi122.iqmcyp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-02T14:54:18.574 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:18 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-02T14:54:18.574 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:18 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-02T14:54:18.574 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:18 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:18.574 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:18 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-02T14:54:18.574 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:18 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-02T14:54:18.574 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:18 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi122.iqmcyp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T14:54:18.574 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:18 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi122.iqmcyp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T14:54:18.574 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:18 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:19.505 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:19 smithi122 ceph-mon[24801]: Saving service nfs.foo spec with placement count:2 2024-04-02T14:54:19.505 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:19 smithi122 ceph-mon[24801]: Creating key for client.nfs.foo.0.0.smithi122.iqmcyp 2024-04-02T14:54:19.505 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:19 smithi122 ceph-mon[24801]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-02T14:54:19.506 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:19 smithi122 ceph-mon[24801]: Rados config object exists: conf-nfs.foo 2024-04-02T14:54:19.506 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:19 smithi122 ceph-mon[24801]: Creating key for client.nfs.foo.0.0.smithi122.iqmcyp-rgw 2024-04-02T14:54:19.506 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:19 smithi122 ceph-mon[24801]: Deploying daemon nfs.foo.0.0.smithi122.iqmcyp on smithi122 2024-04-02T14:54:19.506 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:19 smithi122 ceph-mon[24801]: osdmap e68: 8 total, 8 up, 8 in 2024-04-02T14:54:19.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:19 smithi183 ceph-mon[27058]: Saving service nfs.foo spec with placement count:2 2024-04-02T14:54:19.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:19 smithi183 ceph-mon[27058]: Creating key for client.nfs.foo.0.0.smithi122.iqmcyp 2024-04-02T14:54:19.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:19 smithi183 ceph-mon[27058]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-02T14:54:19.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:19 smithi183 ceph-mon[27058]: Rados config object exists: conf-nfs.foo 2024-04-02T14:54:19.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:19 smithi183 ceph-mon[27058]: Creating key for client.nfs.foo.0.0.smithi122.iqmcyp-rgw 2024-04-02T14:54:19.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:19 smithi183 ceph-mon[27058]: Deploying daemon nfs.foo.0.0.smithi122.iqmcyp on smithi122 2024-04-02T14:54:19.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:19 smithi183 ceph-mon[27058]: osdmap e68: 8 total, 8 up, 8 in 2024-04-02T14:54:20.566 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:20 smithi122 ceph-mon[24801]: pgmap v231: 97 pgs: 12 creating+peering, 10 unknown, 75 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:54:20.567 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:20 smithi122 ceph-mon[24801]: mgrmap e20: smithi122.szjkfg(active, since 5m), standbys: smithi183.smxedq 2024-04-02T14:54:20.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:20 smithi183 ceph-mon[27058]: pgmap v231: 97 pgs: 12 creating+peering, 10 unknown, 75 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:54:20.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:20 smithi183 ceph-mon[27058]: mgrmap e20: smithi122.szjkfg(active, since 5m), standbys: smithi183.smxedq 2024-04-02T14:54:21.566 INFO:teuthology.orchestra.run.smithi122.stderr:Invalid command: Unexpected argument '--clusterid' 2024-04-02T14:54:21.566 INFO:teuthology.orchestra.run.smithi122.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-04-02T14:54:21.566 INFO:teuthology.orchestra.run.smithi122.stderr:Error EINVAL: invalid command 2024-04-02T14:54:21.921 INFO:teuthology.orchestra.run.smithi122.stdout:{ 2024-04-02T14:54:21.921 INFO:teuthology.orchestra.run.smithi122.stdout: "bind": "/fake", 2024-04-02T14:54:21.921 INFO:teuthology.orchestra.run.smithi122.stdout: "fs": "foofs", 2024-04-02T14:54:21.921 INFO:teuthology.orchestra.run.smithi122.stdout: "path": "/", 2024-04-02T14:54:21.921 INFO:teuthology.orchestra.run.smithi122.stdout: "cluster": "foo", 2024-04-02T14:54:21.921 INFO:teuthology.orchestra.run.smithi122.stdout: "mode": "RW" 2024-04-02T14:54:21.921 INFO:teuthology.orchestra.run.smithi122.stdout:} 2024-04-02T14:54:22.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:22 smithi122 ceph-mon[24801]: pgmap v232: 97 pgs: 12 creating+peering, 85 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:54:22.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:22 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:22.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:22 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi183.mxktoi", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-02T14:54:22.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:22 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi183.mxktoi", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-02T14:54:22.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:22 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-02T14:54:22.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:22 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-02T14:54:22.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:22 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:22.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:22 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-02T14:54:22.589 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:22 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-02T14:54:22.647 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-04-02T14:54:22.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:22 smithi183 ceph-mon[27058]: pgmap v232: 97 pgs: 12 creating+peering, 85 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-02T14:54:22.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:22 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:22.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:22 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi183.mxktoi", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-02T14:54:22.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:22 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi183.mxktoi", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-02T14:54:22.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:22 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-02T14:54:22.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:22 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-02T14:54:22.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:22 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:22.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:22 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-02T14:54:22.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:22 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-02T14:54:23.513 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:23 smithi122 ceph-mon[24801]: Creating key for client.nfs.foo.1.0.smithi183.mxktoi 2024-04-02T14:54:23.513 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:23 smithi122 ceph-mon[24801]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-02T14:54:23.513 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:23 smithi122 ceph-mon[24801]: from='client.14516 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:23.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:23 smithi183 ceph-mon[27058]: Creating key for client.nfs.foo.1.0.smithi183.mxktoi 2024-04-02T14:54:23.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:23 smithi183 ceph-mon[27058]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-02T14:54:23.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:23 smithi183 ceph-mon[27058]: from='client.14516 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:24.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:24 smithi122 ceph-mon[24801]: pgmap v233: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.7 KiB/s wr, 3 op/s 2024-04-02T14:54:24.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:24 smithi183 ceph-mon[27058]: pgmap v233: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.7 KiB/s wr, 3 op/s 2024-04-02T14:54:25.838 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:25 smithi122 ceph-mon[24801]: from='client.14532 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:25.838 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:25 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-02T14:54:25.838 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:25 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-02T14:54:25.838 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:25 smithi122 ceph-mon[24801]: Rados config object exists: conf-nfs.foo 2024-04-02T14:54:25.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:25 smithi122 ceph-mon[24801]: Creating key for client.nfs.foo.1.0.smithi183.mxktoi-rgw 2024-04-02T14:54:25.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:25 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi183.mxktoi-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T14:54:25.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:25 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi183.mxktoi-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T14:54:25.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:25 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:25.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:25 smithi122 ceph-mon[24801]: Deploying daemon nfs.foo.1.0.smithi183.mxktoi on smithi183 2024-04-02T14:54:25.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:25 smithi122 ceph-mon[24801]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-04-02T14:54:25.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:25 smithi183 ceph-mon[27058]: from='client.14532 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:25.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:25 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-02T14:54:25.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:25 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-02T14:54:25.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:25 smithi183 ceph-mon[27058]: Rados config object exists: conf-nfs.foo 2024-04-02T14:54:25.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:25 smithi183 ceph-mon[27058]: Creating key for client.nfs.foo.1.0.smithi183.mxktoi-rgw 2024-04-02T14:54:25.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:25 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi183.mxktoi-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T14:54:25.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:25 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi183.mxktoi-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T14:54:25.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:25 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:25.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:25 smithi183 ceph-mon[27058]: Deploying daemon nfs.foo.1.0.smithi183.mxktoi on smithi183 2024-04-02T14:54:25.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:25 smithi183 ceph-mon[27058]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-04-02T14:54:27.088 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:26 smithi122 ceph-mon[24801]: from='client.14546 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:27.147 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:26 smithi183 ceph-mon[27058]: from='client.14546 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:28.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:28 smithi122 ceph-mon[24801]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-04-02T14:54:28.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:28 smithi122 ceph-mon[24801]: from='client.14550 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:28.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:28 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:28.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:28 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:54:28.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:28 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:28.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:28 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:54:28.618 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:28 smithi183 ceph-mon[27058]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-04-02T14:54:28.618 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:28 smithi183 ceph-mon[27058]: from='client.14550 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:28.618 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:28 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:28.618 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:28 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:54:28.618 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:28 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:28.619 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:28 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:54:29.522 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [7, 2]}]: dispatch 2024-04-02T14:54:29.523 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2024-04-02T14:54:29.523 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-04-02T14:54:29.523 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-02T14:54:29.523 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:54:29.523 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/trash_purge_schedule"}]: dispatch 2024-04-02T14:54:29.523 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:29.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [7, 2]}]: dispatch 2024-04-02T14:54:29.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2024-04-02T14:54:29.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-04-02T14:54:29.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-02T14:54:29.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:54:29.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/trash_purge_schedule"}]: dispatch 2024-04-02T14:54:29.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:30.515 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:30 smithi122 ceph-mon[24801]: from='client.14566 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:30.515 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:30 smithi122 ceph-mon[24801]: pgmap v236: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.5 KiB/s wr, 7 op/s 2024-04-02T14:54:30.515 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [7, 2]}]': finished 2024-04-02T14:54:30.515 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-04-02T14:54:30.515 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-04-02T14:54:30.515 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-02T14:54:30.515 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:30 smithi122 ceph-mon[24801]: osdmap e69: 8 total, 8 up, 8 in 2024-04-02T14:54:30.515 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:30 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:30.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:30 smithi183 ceph-mon[27058]: from='client.14566 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:30.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:30 smithi183 ceph-mon[27058]: pgmap v236: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.5 KiB/s wr, 7 op/s 2024-04-02T14:54:30.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:30 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [7, 2]}]': finished 2024-04-02T14:54:30.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:30 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-04-02T14:54:30.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:30 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-04-02T14:54:30.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:30 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-02T14:54:30.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:30 smithi183 ceph-mon[27058]: osdmap e69: 8 total, 8 up, 8 in 2024-04-02T14:54:30.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:30 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:31.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:31 smithi122 ceph-mon[24801]: from='client.14570 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:31.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:31 smithi122 ceph-mon[24801]: osdmap e70: 8 total, 8 up, 8 in 2024-04-02T14:54:31.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:31 smithi183 ceph-mon[27058]: from='client.14570 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:31.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:31 smithi183 ceph-mon[27058]: osdmap e70: 8 total, 8 up, 8 in 2024-04-02T14:54:32.514 INFO:teuthology.orchestra.run.smithi122.stdout:nfs.foo ?:2049 2/2 2s ago 14s count:2 2024-04-02T14:54:32.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:32 smithi122 ceph-mon[24801]: from='client.14574 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:32.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:32 smithi122 ceph-mon[24801]: pgmap v239: 97 pgs: 4 peering, 93 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-04-02T14:54:32.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:32.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:32.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:32.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:54:32.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:32.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:54:32.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:32.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:32 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:32.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:32 smithi183 ceph-mon[27058]: from='client.14574 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:32.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:32 smithi183 ceph-mon[27058]: pgmap v239: 97 pgs: 4 peering, 93 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-04-02T14:54:32.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:32 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:32.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:32 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:32.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:32 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:32.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:32 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:54:32.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:32 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:32.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:32 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:54:32.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:32 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:32.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:32 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:33.267 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-02T14:54:33.278 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi122.front.sepia.ceph.com 2024-04-02T14:54:33.278 DEBUG:teuthology.orchestra.run.smithi122:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-04-02T14:54:33.308 INFO:teuthology.orchestra.run.smithi122.stderr:+ mkdir /mnt/foo 2024-04-02T14:54:33.323 DEBUG:teuthology.orchestra.run.smithi122:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-04-02T14:54:33.389 INFO:teuthology.orchestra.run.smithi122.stderr:++ hostname 2024-04-02T14:54:33.389 INFO:teuthology.orchestra.run.smithi122.stderr:+ mount -t nfs smithi122:/fake /mnt/foo -o sync 2024-04-02T14:54:33.762 DEBUG:teuthology.orchestra.run.smithi122:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-04-02T14:54:33.829 INFO:teuthology.orchestra.run.smithi122.stderr:+ echo test 2024-04-02T14:54:33.844 DEBUG:teuthology.orchestra.run.smithi122:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-04-02T14:54:33.911 INFO:teuthology.orchestra.run.smithi122.stderr:+ sync 2024-04-02T14:54:34.069 INFO:teuthology.run_tasks:Running task parallel... 2024-04-02T14:54:34.080 INFO:teuthology.task.parallel:starting parallel... 2024-04-02T14:54:34.080 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-02T14:54:34.081 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-02T14:54:34.081 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi122.front.sepia.ceph.com 2024-04-02T14:54:34.081 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-02T14:54:34.082 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-02T14:54:34.082 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-04-02T14:54:34.083 INFO:teuthology.task.exec:Executing custom commands... 2024-04-02T14:54:34.083 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi122.front.sepia.ceph.com 2024-04-02T14:54:34.083 DEBUG:teuthology.orchestra.run.smithi122:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-04-02T14:54:34.160 INFO:teuthology.orchestra.run.smithi122.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-04-02T14:54:34.160 INFO:teuthology.orchestra.run.smithi122.stdout: 2024-04-02T14:54:34.161 INFO:teuthology.orchestra.run.smithi122.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-04-02T14:54:34.169 INFO:teuthology.orchestra.run.smithi122.stdout:3 of 5 processes prepared for launch 0 sec 2024-04-02T14:54:34.269 INFO:teuthology.orchestra.run.smithi122.stdout:5 of 5 processes prepared for launch 0 sec 2024-04-02T14:54:34.269 INFO:teuthology.orchestra.run.smithi122.stdout:releasing clients 2024-04-02T14:54:34.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:34 smithi122 ceph-mon[24801]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:34.588 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:34 smithi122 ceph-mon[24801]: pgmap v240: 97 pgs: 4 peering, 93 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-04-02T14:54:34.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:34 smithi183 ceph-mon[27058]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:34.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:34 smithi183 ceph-mon[27058]: pgmap v240: 97 pgs: 4 peering, 93 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-04-02T14:54:35.269 INFO:teuthology.orchestra.run.smithi122.stdout: 5 158 33.04 MB/sec warmup 1 sec latency 31.543 ms 2024-04-02T14:54:35.636 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:35 smithi122 ceph-mon[24801]: pgmap v241: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 KiB/s wr, 6 op/s; 0 B/s, 0 objects/s recovering 2024-04-02T14:54:35.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:35 smithi183 ceph-mon[27058]: pgmap v241: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 KiB/s wr, 6 op/s; 0 B/s, 0 objects/s recovering 2024-04-02T14:54:36.269 INFO:teuthology.orchestra.run.smithi122.stdout: 5 294 27.22 MB/sec warmup 2 sec latency 31.410 ms 2024-04-02T14:54:37.269 INFO:teuthology.orchestra.run.smithi122.stdout: 5 428 25.12 MB/sec warmup 3 sec latency 45.689 ms 2024-04-02T14:54:37.559 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-02T14:54:38.269 INFO:teuthology.orchestra.run.smithi122.stdout: 5 564 24.19 MB/sec warmup 4 sec latency 34.426 ms 2024-04-02T14:54:38.473 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:38 smithi122 ceph-mon[24801]: pgmap v242: 97 pgs: 97 active+clean; 7.2 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 853 KiB/s wr, 15 op/s; 0 B/s, 0 objects/s recovering 2024-04-02T14:54:38.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:38 smithi183 ceph-mon[27058]: pgmap v242: 97 pgs: 97 active+clean; 7.2 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 853 KiB/s wr, 15 op/s; 0 B/s, 0 objects/s recovering 2024-04-02T14:54:39.269 INFO:teuthology.orchestra.run.smithi122.stdout: 5 874 21.63 MB/sec warmup 5 sec latency 85.899 ms 2024-04-02T14:54:40.269 INFO:teuthology.orchestra.run.smithi122.stdout: 5 1391 21.19 MB/sec warmup 6 sec latency 62.809 ms 2024-04-02T14:54:40.492 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:40 smithi122 ceph-mon[24801]: pgmap v243: 97 pgs: 97 active+clean; 70 MiB data, 116 MiB used, 715 GiB / 715 GiB avail; 206 B/s rd, 7.0 MiB/s wr, 289 op/s; 0 B/s, 0 objects/s recovering 2024-04-02T14:54:40.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:40 smithi183 ceph-mon[27058]: pgmap v243: 97 pgs: 97 active+clean; 70 MiB data, 116 MiB used, 715 GiB / 715 GiB avail; 206 B/s rd, 7.0 MiB/s wr, 289 op/s; 0 B/s, 0 objects/s recovering 2024-04-02T14:54:41.269 INFO:teuthology.orchestra.run.smithi122.stdout: 5 2388 19.94 MB/sec warmup 7 sec latency 18.026 ms 2024-04-02T14:54:41.647 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-02T14:54:42.269 INFO:teuthology.orchestra.run.smithi122.stdout: 5 3251 21.75 MB/sec warmup 8 sec latency 14.442 ms 2024-04-02T14:54:42.511 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:42 smithi122 ceph-mon[24801]: pgmap v244: 97 pgs: 97 active+clean; 97 MiB data, 145 MiB used, 715 GiB / 715 GiB avail; 187 B/s rd, 9.1 MiB/s wr, 348 op/s; 0 B/s, 0 objects/s recovering 2024-04-02T14:54:42.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:42 smithi183 ceph-mon[27058]: pgmap v244: 97 pgs: 97 active+clean; 97 MiB data, 145 MiB used, 715 GiB / 715 GiB avail; 187 B/s rd, 9.1 MiB/s wr, 348 op/s; 0 B/s, 0 objects/s recovering 2024-04-02T14:54:43.269 INFO:teuthology.orchestra.run.smithi122.stdout: 5 3874 22.16 MB/sec warmup 9 sec latency 21.969 ms 2024-04-02T14:54:44.269 INFO:teuthology.orchestra.run.smithi122.stdout: 5 4521 20.70 MB/sec warmup 10 sec latency 26.189 ms 2024-04-02T14:54:44.534 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:44 smithi122 ceph-mon[24801]: pgmap v245: 97 pgs: 97 active+clean; 115 MiB data, 179 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 10 MiB/s wr, 490 op/s; 0 B/s, 0 objects/s recovering 2024-04-02T14:54:44.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:44 smithi183 ceph-mon[27058]: pgmap v245: 97 pgs: 97 active+clean; 115 MiB data, 179 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 10 MiB/s wr, 490 op/s; 0 B/s, 0 objects/s recovering 2024-04-02T14:54:45.271 INFO:teuthology.orchestra.run.smithi122.stdout: 5 5465 20.72 MB/sec warmup 11 sec latency 45.398 ms 2024-04-02T14:54:45.812 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:45 smithi122 ceph-mon[24801]: pgmap v246: 97 pgs: 97 active+clean; 123 MiB data, 209 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 12 MiB/s wr, 618 op/s; 0 B/s, 0 objects/s recovering 2024-04-02T14:54:45.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:45 smithi183 ceph-mon[27058]: pgmap v246: 97 pgs: 97 active+clean; 123 MiB data, 209 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 12 MiB/s wr, 618 op/s; 0 B/s, 0 objects/s recovering 2024-04-02T14:54:46.269 INFO:teuthology.orchestra.run.smithi122.stdout: 5 6123 20.16 MB/sec warmup 12 sec latency 21.985 ms 2024-04-02T14:54:47.001 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph mgr module enable nfs --force' 2024-04-02T14:54:47.269 INFO:teuthology.orchestra.run.smithi122.stdout: 5 6956 21.52 MB/sec warmup 13 sec latency 42.115 ms 2024-04-02T14:54:48.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 7545 21.12 MB/sec warmup 14 sec latency 24.979 ms 2024-04-02T14:54:48.342 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:48 smithi122 ceph-mon[24801]: pgmap v247: 97 pgs: 97 active+clean; 129 MiB data, 241 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 13 MiB/s wr, 734 op/s 2024-04-02T14:54:48.600 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:48 smithi183 ceph-mon[27058]: pgmap v247: 97 pgs: 97 active+clean; 129 MiB data, 241 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 13 MiB/s wr, 734 op/s 2024-04-02T14:54:49.269 INFO:teuthology.orchestra.run.smithi122.stdout: 5 8109 20.21 MB/sec warmup 15 sec latency 30.502 ms 2024-04-02T14:54:50.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 8897 20.17 MB/sec warmup 16 sec latency 31.466 ms 2024-04-02T14:54:50.363 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:50 smithi122 ceph-mon[24801]: pgmap v248: 97 pgs: 97 active+clean; 141 MiB data, 292 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 16 MiB/s wr, 1.01k op/s 2024-04-02T14:54:50.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:50 smithi183 ceph-mon[27058]: pgmap v248: 97 pgs: 97 active+clean; 141 MiB data, 292 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 16 MiB/s wr, 1.01k op/s 2024-04-02T14:54:51.269 INFO:teuthology.orchestra.run.smithi122.stdout: 5 9669 20.02 MB/sec warmup 17 sec latency 22.967 ms 2024-04-02T14:54:51.317 INFO:teuthology.orchestra.run.smithi122.stderr:module 'nfs' is already enabled 2024-04-02T14:54:51.371 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:51 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/520562572' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-02T14:54:51.371 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:51 smithi122 ceph-mon[24801]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-02T14:54:51.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:51 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/520562572' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-02T14:54:51.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:51 smithi183 ceph-mon[27058]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-02T14:54:52.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 10644 21.10 MB/sec warmup 18 sec latency 22.745 ms 2024-04-02T14:54:52.383 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:52 smithi122 ceph-mon[24801]: pgmap v249: 97 pgs: 97 active+clean; 146 MiB data, 328 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 12 MiB/s wr, 987 op/s 2024-04-02T14:54:52.383 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:52 smithi122 ceph-mon[24801]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-04-02T14:54:52.383 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:52 smithi122 ceph-mon[24801]: mgrmap e21: smithi122.szjkfg(active, since 6m), standbys: smithi183.smxedq 2024-04-02T14:54:52.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:52 smithi183 ceph-mon[27058]: pgmap v249: 97 pgs: 97 active+clean; 146 MiB data, 328 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 12 MiB/s wr, 987 op/s 2024-04-02T14:54:52.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:52 smithi183 ceph-mon[27058]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-04-02T14:54:52.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:52 smithi183 ceph-mon[27058]: mgrmap e21: smithi122.szjkfg(active, since 6m), standbys: smithi183.smxedq 2024-04-02T14:54:52.652 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-02T14:54:53.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 11250 20.63 MB/sec warmup 19 sec latency 22.472 ms 2024-04-02T14:54:54.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 11786 20.03 MB/sec warmup 20 sec latency 51.496 ms 2024-04-02T14:54:54.408 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:54 smithi122 ceph-mon[24801]: pgmap v250: 97 pgs: 97 active+clean; 145 MiB data, 372 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 11 MiB/s wr, 984 op/s 2024-04-02T14:54:54.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:54 smithi183 ceph-mon[27058]: pgmap v250: 97 pgs: 97 active+clean; 145 MiB data, 372 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 11 MiB/s wr, 984 op/s 2024-04-02T14:54:55.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 12592 19.96 MB/sec warmup 21 sec latency 35.913 ms 2024-04-02T14:54:55.677 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:55 smithi122 ceph-mon[24801]: pgmap v251: 97 pgs: 97 active+clean; 150 MiB data, 413 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 11 MiB/s wr, 920 op/s 2024-04-02T14:54:55.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:55 smithi183 ceph-mon[27058]: pgmap v251: 97 pgs: 97 active+clean; 150 MiB data, 413 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 11 MiB/s wr, 920 op/s 2024-04-02T14:54:56.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 13171 19.56 MB/sec warmup 22 sec latency 53.067 ms 2024-04-02T14:54:56.357 INFO:teuthology.orchestra.run.smithi122.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3f3f3ed992429b85be729743ad3b39d52dd90744 2024-04-02T14:54:57.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 13905 20.06 MB/sec warmup 23 sec latency 30.020 ms 2024-04-02T14:54:57.434 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:57 smithi122 ceph-mon[24801]: from='client.14596 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3f3f3ed992429b85be729743ad3b39d52dd90744", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:57.434 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:57 smithi122 ceph-mon[24801]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3f3f3ed992429b85be729743ad3b39d52dd90744 2024-04-02T14:54:57.434 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:57 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:57.434 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:57 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:54:57.434 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:57 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:57.434 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:57 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:54:57.434 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:57 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:57.435 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:57 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:57.582 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-02T14:54:57.583 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi122.front.sepia.ceph.com 2024-04-02T14:54:57.583 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-04-02T14:54:57.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:57 smithi183 ceph-mon[27058]: from='client.14596 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3f3f3ed992429b85be729743ad3b39d52dd90744", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:54:57.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:57 smithi183 ceph-mon[27058]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3f3f3ed992429b85be729743ad3b39d52dd90744 2024-04-02T14:54:57.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:57 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:57.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:57 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:54:57.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:57 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:54:57.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:57 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:54:57.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:57 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:57.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:57 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:54:58.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 14472 20.20 MB/sec warmup 24 sec latency 17.085 ms 2024-04-02T14:54:58.447 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:58 smithi122 ceph-mon[24801]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3f3f3ed992429b85be729743ad3b39d52dd90744 2024-04-02T14:54:58.447 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:54:58 smithi122 ceph-mon[24801]: pgmap v252: 97 pgs: 97 active+clean; 154 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 915 op/s 2024-04-02T14:54:58.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:58 smithi183 ceph-mon[27058]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3f3f3ed992429b85be729743ad3b39d52dd90744 2024-04-02T14:54:58.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:54:58 smithi183 ceph-mon[27058]: pgmap v252: 97 pgs: 97 active+clean; 154 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 915 op/s 2024-04-02T14:54:59.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 14850 19.53 MB/sec warmup 25 sec latency 20.460 ms 2024-04-02T14:55:00.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 15333 19.09 MB/sec warmup 26 sec latency 26.975 ms 2024-04-02T14:55:00.475 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:00 smithi122 ceph-mon[24801]: pgmap v253: 97 pgs: 97 active+clean; 159 MiB data, 507 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-04-02T14:55:00.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:00 smithi183 ceph-mon[27058]: pgmap v253: 97 pgs: 97 active+clean; 159 MiB data, 507 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-04-02T14:55:01.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 16096 19.05 MB/sec warmup 27 sec latency 20.315 ms 2024-04-02T14:55:01.420 INFO:teuthology.orchestra.run.smithi122.stdout:true 2024-04-02T14:55:02.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 16705 18.78 MB/sec warmup 28 sec latency 32.477 ms 2024-04-02T14:55:02.485 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:02 smithi122 ceph-mon[24801]: pgmap v254: 97 pgs: 97 active+clean; 164 MiB data, 507 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 9.4 MiB/s wr, 864 op/s 2024-04-02T14:55:02.486 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:02 smithi122 ceph-mon[24801]: from='client.14600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:55:02.497 INFO:teuthology.orchestra.run.smithi122.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T14:55:02.497 INFO:teuthology.orchestra.run.smithi122.stdout:alertmanager.smithi122 smithi122 *:9093,9094 running (5m) 30s ago 7m 24.3M - ba2b418f427c 87ea6de784f7 2024-04-02T14:55:02.497 INFO:teuthology.orchestra.run.smithi122.stdout:crash.smithi122 smithi122 running (6m) 30s ago 6m 6400k - 17.2.0 e1d6a67b021e 140dc3b57d86 2024-04-02T14:55:02.497 INFO:teuthology.orchestra.run.smithi122.stdout:crash.smithi183 smithi183 running (6m) 32s ago 6m 6400k - 17.2.0 e1d6a67b021e 1262330a9cca 2024-04-02T14:55:02.497 INFO:teuthology.orchestra.run.smithi122.stdout:grafana.smithi122 smithi122 *:3000 running (5m) 30s ago 6m 52.6M - 8.3.5 dad864ee21e9 f0623c4719c2 2024-04-02T14:55:02.497 INFO:teuthology.orchestra.run.smithi122.stdout:mds.foofs.smithi122.cghwpx smithi122 running (59s) 30s ago 58s 13.4M - 17.2.0 e1d6a67b021e 5161b53cd4a7 2024-04-02T14:55:02.498 INFO:teuthology.orchestra.run.smithi122.stdout:mds.foofs.smithi183.idnham smithi183 running (56s) 32s ago 56s 14.3M - 17.2.0 e1d6a67b021e 25b5a64c8ee4 2024-04-02T14:55:02.498 INFO:teuthology.orchestra.run.smithi122.stdout:mgr.smithi122.szjkfg smithi122 *:9283 running (8m) 30s ago 8m 459M - 17.2.0 e1d6a67b021e 7583b48bda3e 2024-04-02T14:55:02.498 INFO:teuthology.orchestra.run.smithi122.stdout:mgr.smithi183.smxedq smithi183 *:8443,9283 running (6m) 32s ago 6m 414M - 17.2.0 e1d6a67b021e c066bca1a154 2024-04-02T14:55:02.498 INFO:teuthology.orchestra.run.smithi122.stdout:mon.smithi122 smithi122 running (8m) 30s ago 8m 45.0M 2048M 17.2.0 e1d6a67b021e d8f91f766e22 2024-04-02T14:55:02.498 INFO:teuthology.orchestra.run.smithi122.stdout:mon.smithi183 smithi183 running (6m) 32s ago 6m 37.8M 2048M 17.2.0 e1d6a67b021e d6d4734135ac 2024-04-02T14:55:02.498 INFO:teuthology.orchestra.run.smithi122.stdout:nfs.foo.0.0.smithi122.iqmcyp smithi122 *:2049 running (41s) 30s ago 40s 46.9M - 4.0 e1d6a67b021e 15c2856887b6 2024-04-02T14:55:02.499 INFO:teuthology.orchestra.run.smithi122.stdout:nfs.foo.1.0.smithi183.mxktoi smithi183 *:2049 running (35s) 32s ago 35s 46.7M - 4.0 e1d6a67b021e 545c7ba6873c 2024-04-02T14:55:02.499 INFO:teuthology.orchestra.run.smithi122.stdout:node-exporter.smithi122 smithi122 *:9100 running (6m) 30s ago 6m 17.5M - 1dbe0e931976 8815d797e9e2 2024-04-02T14:55:02.499 INFO:teuthology.orchestra.run.smithi122.stdout:node-exporter.smithi183 smithi183 *:9100 running (6m) 32s ago 6m 17.5M - 1dbe0e931976 6c7613782142 2024-04-02T14:55:02.499 INFO:teuthology.orchestra.run.smithi122.stdout:osd.0 smithi122 running (5m) 30s ago 5m 42.1M 3216M 17.2.0 e1d6a67b021e bee30cdc6385 2024-04-02T14:55:02.499 INFO:teuthology.orchestra.run.smithi122.stdout:osd.1 smithi122 running (4m) 30s ago 4m 42.5M 3216M 17.2.0 e1d6a67b021e 92d52898245e 2024-04-02T14:55:02.499 INFO:teuthology.orchestra.run.smithi122.stdout:osd.2 smithi122 running (4m) 30s ago 4m 41.5M 3216M 17.2.0 e1d6a67b021e b88321d239b1 2024-04-02T14:55:02.499 INFO:teuthology.orchestra.run.smithi122.stdout:osd.3 smithi122 running (4m) 30s ago 4m 42.8M 3216M 17.2.0 e1d6a67b021e 1aec61f33b52 2024-04-02T14:55:02.500 INFO:teuthology.orchestra.run.smithi122.stdout:osd.4 smithi183 running (3m) 32s ago 3m 39.8M 3984M 17.2.0 e1d6a67b021e 0d1dbe82a342 2024-04-02T14:55:02.500 INFO:teuthology.orchestra.run.smithi122.stdout:osd.5 smithi183 running (3m) 32s ago 3m 42.7M 3984M 17.2.0 e1d6a67b021e 9bc6c62fa5cb 2024-04-02T14:55:02.500 INFO:teuthology.orchestra.run.smithi122.stdout:osd.6 smithi183 running (2m) 32s ago 2m 39.4M 3984M 17.2.0 e1d6a67b021e f9c8a0a7bfdc 2024-04-02T14:55:02.500 INFO:teuthology.orchestra.run.smithi122.stdout:osd.7 smithi183 running (2m) 32s ago 2m 44.3M 3984M 17.2.0 e1d6a67b021e 1406cf2e45a8 2024-04-02T14:55:02.500 INFO:teuthology.orchestra.run.smithi122.stdout:prometheus.smithi122 smithi122 *:9095 running (5m) 30s ago 5m 61.9M - 514e6a882f6e be604b0d0ed3 2024-04-02T14:55:02.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:02 smithi183 ceph-mon[27058]: pgmap v254: 97 pgs: 97 active+clean; 164 MiB data, 507 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 9.4 MiB/s wr, 864 op/s 2024-04-02T14:55:02.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:02 smithi183 ceph-mon[27058]: from='client.14600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:55:03.155 INFO:teuthology.orchestra.run.smithi122.stdout:{ 2024-04-02T14:55:03.155 INFO:teuthology.orchestra.run.smithi122.stdout: "mon": { 2024-04-02T14:55:03.155 INFO:teuthology.orchestra.run.smithi122.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T14:55:03.155 INFO:teuthology.orchestra.run.smithi122.stdout: }, 2024-04-02T14:55:03.155 INFO:teuthology.orchestra.run.smithi122.stdout: "mgr": { 2024-04-02T14:55:03.155 INFO:teuthology.orchestra.run.smithi122.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T14:55:03.156 INFO:teuthology.orchestra.run.smithi122.stdout: }, 2024-04-02T14:55:03.156 INFO:teuthology.orchestra.run.smithi122.stdout: "osd": { 2024-04-02T14:55:03.156 INFO:teuthology.orchestra.run.smithi122.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T14:55:03.156 INFO:teuthology.orchestra.run.smithi122.stdout: }, 2024-04-02T14:55:03.156 INFO:teuthology.orchestra.run.smithi122.stdout: "mds": { 2024-04-02T14:55:03.156 INFO:teuthology.orchestra.run.smithi122.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T14:55:03.156 INFO:teuthology.orchestra.run.smithi122.stdout: }, 2024-04-02T14:55:03.156 INFO:teuthology.orchestra.run.smithi122.stdout: "overall": { 2024-04-02T14:55:03.156 INFO:teuthology.orchestra.run.smithi122.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-02T14:55:03.156 INFO:teuthology.orchestra.run.smithi122.stdout: } 2024-04-02T14:55:03.156 INFO:teuthology.orchestra.run.smithi122.stdout:} 2024-04-02T14:55:03.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 17302 19.01 MB/sec warmup 29 sec latency 26.078 ms 2024-04-02T14:55:03.491 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:03 smithi122 ceph-mon[24801]: from='client.24351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:55:03.491 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:03 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/1536934444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T14:55:03.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:03 smithi183 ceph-mon[27058]: from='client.24351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:55:03.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:03 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/1536934444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T14:55:03.742 INFO:teuthology.orchestra.run.smithi122.stdout:{ 2024-04-02T14:55:03.742 INFO:teuthology.orchestra.run.smithi122.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3f3f3ed992429b85be729743ad3b39d52dd90744", 2024-04-02T14:55:03.743 INFO:teuthology.orchestra.run.smithi122.stdout: "in_progress": true, 2024-04-02T14:55:03.743 INFO:teuthology.orchestra.run.smithi122.stdout: "services_complete": [], 2024-04-02T14:55:03.743 INFO:teuthology.orchestra.run.smithi122.stdout: "progress": "", 2024-04-02T14:55:03.743 INFO:teuthology.orchestra.run.smithi122.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:3f3f3ed992429b85be729743ad3b39d52dd90744 image" 2024-04-02T14:55:03.743 INFO:teuthology.orchestra.run.smithi122.stdout:} 2024-04-02T14:55:04.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 17798 19.09 MB/sec warmup 30 sec latency 27.726 ms 2024-04-02T14:55:04.409 INFO:teuthology.orchestra.run.smithi122.stdout:HEALTH_OK 2024-04-02T14:55:04.502 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:04 smithi122 ceph-mon[24801]: from='client.14608 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:55:04.502 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:04 smithi122 ceph-mon[24801]: pgmap v255: 97 pgs: 97 active+clean; 164 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 813 op/s 2024-04-02T14:55:04.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:04 smithi183 ceph-mon[27058]: from='client.14608 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:55:04.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:04 smithi183 ceph-mon[27058]: pgmap v255: 97 pgs: 97 active+clean; 164 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 813 op/s 2024-04-02T14:55:05.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 18255 18.80 MB/sec warmup 31 sec latency 37.814 ms 2024-04-02T14:55:05.507 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:05 smithi122 ceph-mon[24801]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:55:05.507 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:05 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/31766313' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T14:55:05.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:05 smithi183 ceph-mon[27058]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:55:05.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:05 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/31766313' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T14:55:06.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 18768 18.45 MB/sec warmup 32 sec latency 91.258 ms 2024-04-02T14:55:06.529 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:06 smithi122 ceph-mon[24801]: pgmap v256: 97 pgs: 97 active+clean; 169 MiB data, 560 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 837 op/s 2024-04-02T14:55:06.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:06 smithi183 ceph-mon[27058]: pgmap v256: 97 pgs: 97 active+clean; 169 MiB data, 560 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 837 op/s 2024-04-02T14:55:07.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 19430 18.46 MB/sec warmup 33 sec latency 48.431 ms 2024-04-02T14:55:08.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 20142 18.14 MB/sec warmup 34 sec latency 39.615 ms 2024-04-02T14:55:08.559 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:08 smithi122 ceph-mon[24801]: pgmap v257: 97 pgs: 97 active+clean; 172 MiB data, 578 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 8.4 MiB/s wr, 840 op/s 2024-04-02T14:55:08.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:08 smithi183 ceph-mon[27058]: pgmap v257: 97 pgs: 97 active+clean; 172 MiB data, 578 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 8.4 MiB/s wr, 840 op/s 2024-04-02T14:55:09.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 20929 18.55 MB/sec warmup 35 sec latency 29.151 ms 2024-04-02T14:55:10.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 21602 18.81 MB/sec warmup 36 sec latency 21.769 ms 2024-04-02T14:55:10.586 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:10 smithi122 ceph-mon[24801]: pgmap v258: 97 pgs: 97 active+clean; 177 MiB data, 640 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 9.3 MiB/s wr, 911 op/s 2024-04-02T14:55:10.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:10 smithi183 ceph-mon[27058]: pgmap v258: 97 pgs: 97 active+clean; 177 MiB data, 640 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 9.3 MiB/s wr, 911 op/s 2024-04-02T14:55:11.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 22017 18.37 MB/sec warmup 37 sec latency 27.262 ms 2024-04-02T14:55:12.270 INFO:teuthology.orchestra.run.smithi122.stdout: 5 22453 18.11 MB/sec warmup 38 sec latency 68.259 ms 2024-04-02T14:55:12.617 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:12 smithi122 ceph-mon[24801]: pgmap v259: 97 pgs: 97 active+clean; 179 MiB data, 669 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 8.0 MiB/s wr, 738 op/s 2024-04-02T14:55:12.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:12 smithi183 ceph-mon[27058]: pgmap v259: 97 pgs: 97 active+clean; 179 MiB data, 669 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 8.0 MiB/s wr, 738 op/s 2024-04-02T14:55:13.279 INFO:teuthology.orchestra.run.smithi122.stdout: 5 23142 18.09 MB/sec warmup 39 sec latency 41.049 ms 2024-04-02T14:55:14.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 23691 17.83 MB/sec warmup 40 sec latency 39.500 ms 2024-04-02T14:55:14.663 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:14 smithi122 ceph-mon[24801]: pgmap v260: 97 pgs: 97 active+clean; 188 MiB data, 685 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.6 MiB/s wr, 806 op/s 2024-04-02T14:55:14.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:14 smithi183 ceph-mon[27058]: pgmap v260: 97 pgs: 97 active+clean; 188 MiB data, 685 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.6 MiB/s wr, 806 op/s 2024-04-02T14:55:15.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 24157 17.88 MB/sec warmup 41 sec latency 31.807 ms 2024-04-02T14:55:15.674 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:15 smithi122 ceph-mon[24801]: pgmap v261: 97 pgs: 97 active+clean; 185 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.0 MiB/s wr, 692 op/s 2024-04-02T14:55:15.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:15 smithi183 ceph-mon[27058]: pgmap v261: 97 pgs: 97 active+clean; 185 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.0 MiB/s wr, 692 op/s 2024-04-02T14:55:16.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 24707 17.95 MB/sec warmup 42 sec latency 31.747 ms 2024-04-02T14:55:17.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 25106 17.96 MB/sec warmup 43 sec latency 34.777 ms 2024-04-02T14:55:18.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 25297 17.59 MB/sec warmup 44 sec latency 60.778 ms 2024-04-02T14:55:18.473 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:18 smithi122 ceph-mon[24801]: pgmap v262: 97 pgs: 97 active+clean; 192 MiB data, 727 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 7.9 MiB/s wr, 806 op/s 2024-04-02T14:55:18.600 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:18 smithi183 ceph-mon[27058]: pgmap v262: 97 pgs: 97 active+clean; 192 MiB data, 727 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 7.9 MiB/s wr, 806 op/s 2024-04-02T14:55:19.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 25728 17.27 MB/sec warmup 45 sec latency 32.493 ms 2024-04-02T14:55:20.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 26138 17.12 MB/sec warmup 46 sec latency 44.839 ms 2024-04-02T14:55:20.501 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:20 smithi122 ceph-mon[24801]: pgmap v263: 97 pgs: 97 active+clean; 192 MiB data, 787 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.4 MiB/s wr, 763 op/s 2024-04-02T14:55:20.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:20 smithi183 ceph-mon[27058]: pgmap v263: 97 pgs: 97 active+clean; 192 MiB data, 787 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.4 MiB/s wr, 763 op/s 2024-04-02T14:55:21.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 26837 17.06 MB/sec warmup 47 sec latency 21.482 ms 2024-04-02T14:55:22.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 27359 16.92 MB/sec warmup 48 sec latency 57.426 ms 2024-04-02T14:55:22.529 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:22 smithi122 ceph-mon[24801]: pgmap v264: 97 pgs: 97 active+clean; 203 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.0 MiB/s wr, 731 op/s 2024-04-02T14:55:22.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:22 smithi183 ceph-mon[27058]: pgmap v264: 97 pgs: 97 active+clean; 203 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.0 MiB/s wr, 731 op/s 2024-04-02T14:55:23.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 28242 17.33 MB/sec warmup 49 sec latency 24.832 ms 2024-04-02T14:55:24.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 28830 17.39 MB/sec warmup 50 sec latency 24.308 ms 2024-04-02T14:55:24.548 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:24 smithi122 ceph-mon[24801]: pgmap v265: 97 pgs: 97 active+clean; 204 MiB data, 815 MiB used, 714 GiB / 715 GiB avail; 8.6 KiB/s rd, 8.0 MiB/s wr, 843 op/s 2024-04-02T14:55:24.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:24 smithi183 ceph-mon[27058]: pgmap v265: 97 pgs: 97 active+clean; 204 MiB data, 815 MiB used, 714 GiB / 715 GiB avail; 8.6 KiB/s rd, 8.0 MiB/s wr, 843 op/s 2024-04-02T14:55:25.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 29342 17.15 MB/sec warmup 51 sec latency 32.423 ms 2024-04-02T14:55:25.836 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:25 smithi122 ceph-mon[24801]: pgmap v266: 97 pgs: 97 active+clean; 202 MiB data, 842 MiB used, 714 GiB / 715 GiB avail; 8.6 KiB/s rd, 7.0 MiB/s wr, 693 op/s 2024-04-02T14:55:25.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:25 smithi183 ceph-mon[27058]: pgmap v266: 97 pgs: 97 active+clean; 202 MiB data, 842 MiB used, 714 GiB / 715 GiB avail; 8.6 KiB/s rd, 7.0 MiB/s wr, 693 op/s 2024-04-02T14:55:26.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 29738 17.04 MB/sec warmup 52 sec latency 29.795 ms 2024-04-02T14:55:27.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 30417 16.94 MB/sec warmup 53 sec latency 32.054 ms 2024-04-02T14:55:28.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 30843 16.78 MB/sec warmup 54 sec latency 38.177 ms 2024-04-02T14:55:28.354 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:28 smithi122 ceph-mon[24801]: pgmap v267: 97 pgs: 97 active+clean; 208 MiB data, 882 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.2 MiB/s wr, 743 op/s 2024-04-02T14:55:28.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:28 smithi183 ceph-mon[27058]: pgmap v267: 97 pgs: 97 active+clean; 208 MiB data, 882 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.2 MiB/s wr, 743 op/s 2024-04-02T14:55:29.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 31561 17.03 MB/sec warmup 55 sec latency 29.183 ms 2024-04-02T14:55:29.358 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:55:29.358 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:29 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/trash_purge_schedule"}]: dispatch 2024-04-02T14:55:29.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/mirror_snapshot_schedule"}]: dispatch 2024-04-02T14:55:29.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:29 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi122.szjkfg/trash_purge_schedule"}]: dispatch 2024-04-02T14:55:30.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 32215 17.20 MB/sec warmup 56 sec latency 23.721 ms 2024-04-02T14:55:30.373 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:30 smithi122 ceph-mon[24801]: pgmap v268: 97 pgs: 97 active+clean; 211 MiB data, 928 MiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 8.3 MiB/s wr, 768 op/s 2024-04-02T14:55:30.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:30 smithi183 ceph-mon[27058]: pgmap v268: 97 pgs: 97 active+clean; 211 MiB data, 928 MiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 8.3 MiB/s wr, 768 op/s 2024-04-02T14:55:31.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 32650 16.96 MB/sec warmup 57 sec latency 36.134 ms 2024-04-02T14:55:31.524 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:31 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:55:31.524 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:31 smithi122 ceph-mon[24801]: Upgrade: Target is version 19.0.0-2592-g3f3f3ed9 (unknown) 2024-04-02T14:55:31.524 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:31 smithi122 ceph-mon[24801]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac, digests ['quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac'] 2024-04-02T14:55:31.524 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:31 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:55:31.524 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:31 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:55:31.524 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:31 smithi122 ceph-mon[24801]: Upgrade: Need to upgrade myself (mgr.smithi122.szjkfg) 2024-04-02T14:55:31.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:31 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:55:31.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:31 smithi183 ceph-mon[27058]: Upgrade: Target is version 19.0.0-2592-g3f3f3ed9 (unknown) 2024-04-02T14:55:31.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:31 smithi183 ceph-mon[27058]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac, digests ['quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac'] 2024-04-02T14:55:31.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:31 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:55:31.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:31 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:55:31.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:31 smithi183 ceph-mon[27058]: Upgrade: Need to upgrade myself (mgr.smithi122.szjkfg) 2024-04-02T14:55:32.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 33126 16.86 MB/sec warmup 58 sec latency 19.015 ms 2024-04-02T14:55:32.541 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:32 smithi122 ceph-mon[24801]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac on smithi183 2024-04-02T14:55:32.541 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:32 smithi122 ceph-mon[24801]: pgmap v269: 97 pgs: 97 active+clean; 209 MiB data, 914 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.6 MiB/s wr, 738 op/s 2024-04-02T14:55:32.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:32 smithi183 ceph-mon[27058]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac on smithi183 2024-04-02T14:55:32.649 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:32 smithi183 ceph-mon[27058]: pgmap v269: 97 pgs: 97 active+clean; 209 MiB data, 914 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.6 MiB/s wr, 738 op/s 2024-04-02T14:55:33.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 33725 16.81 MB/sec warmup 59 sec latency 27.527 ms 2024-04-02T14:55:34.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 34349 16.67 MB/sec warmup 60 sec latency 41.178 ms 2024-04-02T14:55:34.555 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:34 smithi122 ceph-mon[24801]: pgmap v270: 97 pgs: 97 active+clean; 228 MiB data, 958 MiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.4 MiB/s wr, 808 op/s 2024-04-02T14:55:34.841 INFO:teuthology.orchestra.run.smithi122.stdout:true 2024-04-02T14:55:34.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:34 smithi183 ceph-mon[27058]: pgmap v270: 97 pgs: 97 active+clean; 228 MiB data, 958 MiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.4 MiB/s wr, 808 op/s 2024-04-02T14:55:35.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 34963 16.86 MB/sec warmup 61 sec latency 21.638 ms 2024-04-02T14:55:35.650 INFO:teuthology.orchestra.run.smithi122.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T14:55:35.650 INFO:teuthology.orchestra.run.smithi122.stdout:alertmanager.smithi122 smithi122 *:9093,9094 running (6m) 63s ago 7m 24.3M - ba2b418f427c 87ea6de784f7 2024-04-02T14:55:35.650 INFO:teuthology.orchestra.run.smithi122.stdout:crash.smithi122 smithi122 running (7m) 63s ago 7m 6400k - 17.2.0 e1d6a67b021e 140dc3b57d86 2024-04-02T14:55:35.650 INFO:teuthology.orchestra.run.smithi122.stdout:crash.smithi183 smithi183 running (6m) 65s ago 6m 6400k - 17.2.0 e1d6a67b021e 1262330a9cca 2024-04-02T14:55:35.650 INFO:teuthology.orchestra.run.smithi122.stdout:grafana.smithi122 smithi122 *:3000 running (5m) 63s ago 7m 52.6M - 8.3.5 dad864ee21e9 f0623c4719c2 2024-04-02T14:55:35.650 INFO:teuthology.orchestra.run.smithi122.stdout:mds.foofs.smithi122.cghwpx smithi122 running (92s) 63s ago 91s 13.4M - 17.2.0 e1d6a67b021e 5161b53cd4a7 2024-04-02T14:55:35.650 INFO:teuthology.orchestra.run.smithi122.stdout:mds.foofs.smithi183.idnham smithi183 running (89s) 65s ago 89s 14.3M - 17.2.0 e1d6a67b021e 25b5a64c8ee4 2024-04-02T14:55:35.650 INFO:teuthology.orchestra.run.smithi122.stdout:mgr.smithi122.szjkfg smithi122 *:9283 running (8m) 63s ago 8m 459M - 17.2.0 e1d6a67b021e 7583b48bda3e 2024-04-02T14:55:35.650 INFO:teuthology.orchestra.run.smithi122.stdout:mgr.smithi183.smxedq smithi183 *:8443,9283 running (6m) 65s ago 6m 414M - 17.2.0 e1d6a67b021e c066bca1a154 2024-04-02T14:55:35.651 INFO:teuthology.orchestra.run.smithi122.stdout:mon.smithi122 smithi122 running (8m) 63s ago 8m 45.0M 2048M 17.2.0 e1d6a67b021e d8f91f766e22 2024-04-02T14:55:35.651 INFO:teuthology.orchestra.run.smithi122.stdout:mon.smithi183 smithi183 running (6m) 65s ago 6m 37.8M 2048M 17.2.0 e1d6a67b021e d6d4734135ac 2024-04-02T14:55:35.651 INFO:teuthology.orchestra.run.smithi122.stdout:nfs.foo.0.0.smithi122.iqmcyp smithi122 *:2049 running (74s) 63s ago 73s 46.9M - 4.0 e1d6a67b021e 15c2856887b6 2024-04-02T14:55:35.651 INFO:teuthology.orchestra.run.smithi122.stdout:nfs.foo.1.0.smithi183.mxktoi smithi183 *:2049 running (68s) 65s ago 68s 46.7M - 4.0 e1d6a67b021e 545c7ba6873c 2024-04-02T14:55:35.651 INFO:teuthology.orchestra.run.smithi122.stdout:node-exporter.smithi122 smithi122 *:9100 running (7m) 63s ago 7m 17.5M - 1dbe0e931976 8815d797e9e2 2024-04-02T14:55:35.651 INFO:teuthology.orchestra.run.smithi122.stdout:node-exporter.smithi183 smithi183 *:9100 running (6m) 65s ago 6m 17.5M - 1dbe0e931976 6c7613782142 2024-04-02T14:55:35.651 INFO:teuthology.orchestra.run.smithi122.stdout:osd.0 smithi122 running (6m) 63s ago 6m 42.1M 3216M 17.2.0 e1d6a67b021e bee30cdc6385 2024-04-02T14:55:35.651 INFO:teuthology.orchestra.run.smithi122.stdout:osd.1 smithi122 running (5m) 63s ago 5m 42.5M 3216M 17.2.0 e1d6a67b021e 92d52898245e 2024-04-02T14:55:35.651 INFO:teuthology.orchestra.run.smithi122.stdout:osd.2 smithi122 running (5m) 63s ago 5m 41.5M 3216M 17.2.0 e1d6a67b021e b88321d239b1 2024-04-02T14:55:35.651 INFO:teuthology.orchestra.run.smithi122.stdout:osd.3 smithi122 running (4m) 63s ago 4m 42.8M 3216M 17.2.0 e1d6a67b021e 1aec61f33b52 2024-04-02T14:55:35.651 INFO:teuthology.orchestra.run.smithi122.stdout:osd.4 smithi183 running (4m) 65s ago 4m 39.8M 3984M 17.2.0 e1d6a67b021e 0d1dbe82a342 2024-04-02T14:55:35.651 INFO:teuthology.orchestra.run.smithi122.stdout:osd.5 smithi183 running (3m) 65s ago 3m 42.7M 3984M 17.2.0 e1d6a67b021e 9bc6c62fa5cb 2024-04-02T14:55:35.651 INFO:teuthology.orchestra.run.smithi122.stdout:osd.6 smithi183 running (3m) 65s ago 3m 39.4M 3984M 17.2.0 e1d6a67b021e f9c8a0a7bfdc 2024-04-02T14:55:35.651 INFO:teuthology.orchestra.run.smithi122.stdout:osd.7 smithi183 running (3m) 65s ago 3m 44.3M 3984M 17.2.0 e1d6a67b021e 1406cf2e45a8 2024-04-02T14:55:35.651 INFO:teuthology.orchestra.run.smithi122.stdout:prometheus.smithi122 smithi122 *:9095 running (6m) 63s ago 6m 61.9M - 514e6a882f6e be604b0d0ed3 2024-04-02T14:55:35.820 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:35 smithi122 ceph-mon[24801]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:55:35.821 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:35 smithi122 ceph-mon[24801]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:55:35.821 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:35 smithi122 ceph-mon[24801]: pgmap v271: 97 pgs: 97 active+clean; 226 MiB data, 1005 MiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.9 MiB/s wr, 679 op/s 2024-04-02T14:55:35.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:35 smithi183 ceph-mon[27058]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:55:35.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:35 smithi183 ceph-mon[27058]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:55:35.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:35 smithi183 ceph-mon[27058]: pgmap v271: 97 pgs: 97 active+clean; 226 MiB data, 1005 MiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.9 MiB/s wr, 679 op/s 2024-04-02T14:55:36.146 INFO:teuthology.orchestra.run.smithi122.stdout:{ 2024-04-02T14:55:36.146 INFO:teuthology.orchestra.run.smithi122.stdout: "mon": { 2024-04-02T14:55:36.146 INFO:teuthology.orchestra.run.smithi122.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T14:55:36.146 INFO:teuthology.orchestra.run.smithi122.stdout: }, 2024-04-02T14:55:36.146 INFO:teuthology.orchestra.run.smithi122.stdout: "mgr": { 2024-04-02T14:55:36.146 INFO:teuthology.orchestra.run.smithi122.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T14:55:36.146 INFO:teuthology.orchestra.run.smithi122.stdout: }, 2024-04-02T14:55:36.146 INFO:teuthology.orchestra.run.smithi122.stdout: "osd": { 2024-04-02T14:55:36.147 INFO:teuthology.orchestra.run.smithi122.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T14:55:36.147 INFO:teuthology.orchestra.run.smithi122.stdout: }, 2024-04-02T14:55:36.147 INFO:teuthology.orchestra.run.smithi122.stdout: "mds": { 2024-04-02T14:55:36.147 INFO:teuthology.orchestra.run.smithi122.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T14:55:36.147 INFO:teuthology.orchestra.run.smithi122.stdout: }, 2024-04-02T14:55:36.147 INFO:teuthology.orchestra.run.smithi122.stdout: "overall": { 2024-04-02T14:55:36.147 INFO:teuthology.orchestra.run.smithi122.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-02T14:55:36.147 INFO:teuthology.orchestra.run.smithi122.stdout: } 2024-04-02T14:55:36.147 INFO:teuthology.orchestra.run.smithi122.stdout:} 2024-04-02T14:55:36.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 35605 17.01 MB/sec warmup 62 sec latency 27.432 ms 2024-04-02T14:55:36.543 INFO:teuthology.orchestra.run.smithi122.stdout:{ 2024-04-02T14:55:36.543 INFO:teuthology.orchestra.run.smithi122.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac", 2024-04-02T14:55:36.543 INFO:teuthology.orchestra.run.smithi122.stdout: "in_progress": true, 2024-04-02T14:55:36.543 INFO:teuthology.orchestra.run.smithi122.stdout: "services_complete": [], 2024-04-02T14:55:36.543 INFO:teuthology.orchestra.run.smithi122.stdout: "progress": "0/23 daemons upgraded", 2024-04-02T14:55:36.543 INFO:teuthology.orchestra.run.smithi122.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac image on host smithi183" 2024-04-02T14:55:36.543 INFO:teuthology.orchestra.run.smithi122.stdout:} 2024-04-02T14:55:36.833 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:36 smithi122 ceph-mon[24801]: from='client.14630 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:55:36.833 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:36 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/3446879324' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T14:55:36.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:36 smithi183 ceph-mon[27058]: from='client.14630 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:55:36.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:36 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/3446879324' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T14:55:37.144 INFO:teuthology.orchestra.run.smithi122.stdout:HEALTH_OK 2024-04-02T14:55:37.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 35924 16.84 MB/sec warmup 63 sec latency 32.713 ms 2024-04-02T14:55:37.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:37 smithi122 ceph-mon[24801]: from='client.14638 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:55:37.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:37 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/1823242386' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T14:55:37.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:37 smithi122 ceph-mon[24801]: pgmap v272: 97 pgs: 97 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.0 MiB/s wr, 763 op/s 2024-04-02T14:55:37.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:37 smithi183 ceph-mon[27058]: from='client.14638 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:55:37.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:37 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/1823242386' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T14:55:37.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:37 smithi183 ceph-mon[27058]: pgmap v272: 97 pgs: 97 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.0 MiB/s wr, 763 op/s 2024-04-02T14:55:38.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 36485 16.66 MB/sec warmup 64 sec latency 32.328 ms 2024-04-02T14:55:39.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 37222 16.73 MB/sec warmup 65 sec latency 34.085 ms 2024-04-02T14:55:40.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 37910 16.62 MB/sec warmup 66 sec latency 44.634 ms 2024-04-02T14:55:40.352 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:40 smithi122 ceph-mon[24801]: pgmap v273: 97 pgs: 97 active+clean; 238 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 889 op/s 2024-04-02T14:55:40.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:40 smithi183 ceph-mon[27058]: pgmap v273: 97 pgs: 97 active+clean; 238 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 889 op/s 2024-04-02T14:55:41.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 38510 16.78 MB/sec warmup 67 sec latency 42.714 ms 2024-04-02T14:55:42.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 39235 16.94 MB/sec warmup 68 sec latency 38.850 ms 2024-04-02T14:55:42.367 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:42 smithi122 ceph-mon[24801]: pgmap v274: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.3 MiB/s wr, 784 op/s 2024-04-02T14:55:42.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:42 smithi183 ceph-mon[27058]: pgmap v274: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.3 MiB/s wr, 784 op/s 2024-04-02T14:55:43.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 39720 16.78 MB/sec warmup 69 sec latency 44.548 ms 2024-04-02T14:55:44.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 40464 16.80 MB/sec warmup 70 sec latency 43.939 ms 2024-04-02T14:55:44.376 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:44 smithi122 ceph-mon[24801]: pgmap v275: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.4 MiB/s wr, 931 op/s 2024-04-02T14:55:44.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:44 smithi183 ceph-mon[27058]: pgmap v275: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.4 MiB/s wr, 931 op/s 2024-04-02T14:55:45.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 41396 16.76 MB/sec warmup 71 sec latency 30.916 ms 2024-04-02T14:55:45.633 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:45 smithi122 ceph-mon[24801]: pgmap v276: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.0 MiB/s wr, 738 op/s 2024-04-02T14:55:45.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:45 smithi183 ceph-mon[27058]: pgmap v276: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.0 MiB/s wr, 738 op/s 2024-04-02T14:55:46.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 42001 16.90 MB/sec warmup 72 sec latency 33.181 ms 2024-04-02T14:55:47.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 42684 17.06 MB/sec warmup 73 sec latency 45.605 ms 2024-04-02T14:55:48.279 INFO:teuthology.orchestra.run.smithi122.stdout: 5 43056 16.92 MB/sec warmup 74 sec latency 28.980 ms 2024-04-02T14:55:48.406 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:48 smithi122 ceph-mon[24801]: pgmap v277: 97 pgs: 97 active+clean; 250 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.3 MiB/s wr, 795 op/s 2024-04-02T14:55:48.600 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:48 smithi183 ceph-mon[27058]: pgmap v277: 97 pgs: 97 active+clean; 250 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.3 MiB/s wr, 795 op/s 2024-04-02T14:55:49.279 INFO:teuthology.orchestra.run.smithi122.stdout: 5 43542 16.77 MB/sec warmup 75 sec latency 34.048 ms 2024-04-02T14:55:50.279 INFO:teuthology.orchestra.run.smithi122.stdout: 5 44216 16.82 MB/sec warmup 76 sec latency 40.188 ms 2024-04-02T14:55:50.347 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:50 smithi122 ceph-mon[24801]: pgmap v278: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.4 KiB/s rd, 10 MiB/s wr, 965 op/s 2024-04-02T14:55:50.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:50 smithi183 ceph-mon[27058]: pgmap v278: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.4 KiB/s rd, 10 MiB/s wr, 965 op/s 2024-04-02T14:55:51.278 INFO:teuthology.orchestra.run.smithi122.stdout: 5 45069 16.74 MB/sec warmup 77 sec latency 23.465 ms 2024-04-02T14:55:52.279 INFO:teuthology.orchestra.run.smithi122.stdout: 5 45746 16.91 MB/sec warmup 78 sec latency 42.521 ms 2024-04-02T14:55:52.359 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:52 smithi122 ceph-mon[24801]: pgmap v279: 97 pgs: 97 active+clean; 263 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 857 op/s 2024-04-02T14:55:52.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:52 smithi183 ceph-mon[27058]: pgmap v279: 97 pgs: 97 active+clean; 263 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 857 op/s 2024-04-02T14:55:53.279 INFO:teuthology.orchestra.run.smithi122.stdout: 5 46251 16.99 MB/sec warmup 79 sec latency 44.956 ms 2024-04-02T14:55:54.279 INFO:teuthology.orchestra.run.smithi122.stdout: 5 46685 16.85 MB/sec warmup 80 sec latency 30.037 ms 2024-04-02T14:55:54.406 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:54 smithi122 ceph-mon[24801]: pgmap v280: 97 pgs: 97 active+clean; 264 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 9.3 MiB/s wr, 839 op/s 2024-04-02T14:55:54.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:54 smithi183 ceph-mon[27058]: pgmap v280: 97 pgs: 97 active+clean; 264 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 9.3 MiB/s wr, 839 op/s 2024-04-02T14:55:55.279 INFO:teuthology.orchestra.run.smithi122.stdout: 5 47216 16.73 MB/sec warmup 81 sec latency 32.490 ms 2024-04-02T14:55:55.687 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:55 smithi122 ceph-mon[24801]: pgmap v281: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.6 MiB/s wr, 792 op/s 2024-04-02T14:55:55.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:55 smithi183 ceph-mon[27058]: pgmap v281: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.6 MiB/s wr, 792 op/s 2024-04-02T14:55:56.286 INFO:teuthology.orchestra.run.smithi122.stdout: 5 47909 16.76 MB/sec warmup 82 sec latency 23.031 ms 2024-04-02T14:55:57.286 INFO:teuthology.orchestra.run.smithi122.stdout: 5 48554 16.66 MB/sec warmup 83 sec latency 30.314 ms 2024-04-02T14:55:58.286 INFO:teuthology.orchestra.run.smithi122.stdout: 5 49215 16.81 MB/sec warmup 84 sec latency 41.552 ms 2024-04-02T14:55:58.480 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:55:58 smithi122 ceph-mon[24801]: pgmap v282: 97 pgs: 97 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.5 MiB/s wr, 794 op/s 2024-04-02T14:55:58.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:55:58 smithi183 ceph-mon[27058]: pgmap v282: 97 pgs: 97 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.5 MiB/s wr, 794 op/s 2024-04-02T14:55:59.286 INFO:teuthology.orchestra.run.smithi122.stdout: 5 49899 16.93 MB/sec warmup 85 sec latency 27.870 ms 2024-04-02T14:56:00.286 INFO:teuthology.orchestra.run.smithi122.stdout: 5 50399 16.80 MB/sec warmup 86 sec latency 24.912 ms 2024-04-02T14:56:00.492 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:00 smithi122 ceph-mon[24801]: pgmap v283: 97 pgs: 97 active+clean; 271 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.6 MiB/s wr, 903 op/s 2024-04-02T14:56:00.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:00 smithi183 ceph-mon[27058]: pgmap v283: 97 pgs: 97 active+clean; 271 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.6 MiB/s wr, 903 op/s 2024-04-02T14:56:01.286 INFO:teuthology.orchestra.run.smithi122.stdout: 5 51052 16.78 MB/sec warmup 87 sec latency 21.362 ms 2024-04-02T14:56:01.462 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:01 smithi183 ceph-mon[27058]: Upgrade: Updating mgr.smithi183.smxedq 2024-04-02T14:56:01.462 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:01 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:56:01.462 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:01 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi183.smxedq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T14:56:01.462 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:01 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T14:56:01.462 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:01 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:56:01.462 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:01 smithi183 ceph-mon[27058]: Deploying daemon mgr.smithi183.smxedq on smithi183 2024-04-02T14:56:01.502 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:01 smithi122 ceph-mon[24801]: Upgrade: Updating mgr.smithi183.smxedq 2024-04-02T14:56:01.503 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:01 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:56:01.503 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:01 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi183.smxedq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T14:56:01.503 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:01 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T14:56:01.503 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:01 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:56:01.503 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:01 smithi122 ceph-mon[24801]: Deploying daemon mgr.smithi183.smxedq on smithi183 2024-04-02T14:56:02.286 INFO:teuthology.orchestra.run.smithi122.stdout: 5 51868 16.75 MB/sec warmup 88 sec latency 26.364 ms 2024-04-02T14:56:02.509 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:02 smithi122 ceph-mon[24801]: pgmap v284: 97 pgs: 97 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.2 MiB/s wr, 769 op/s 2024-04-02T14:56:02.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:02 smithi183 ceph-mon[27058]: pgmap v284: 97 pgs: 97 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.2 MiB/s wr, 769 op/s 2024-04-02T14:56:03.286 INFO:teuthology.orchestra.run.smithi122.stdout: 5 52501 16.81 MB/sec warmup 89 sec latency 47.481 ms 2024-04-02T14:56:04.286 INFO:teuthology.orchestra.run.smithi122.stdout: 5 53172 16.92 MB/sec warmup 90 sec latency 52.360 ms 2024-04-02T14:56:04.519 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:04 smithi122 ceph-mon[24801]: pgmap v285: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.7 MiB/s wr, 817 op/s 2024-04-02T14:56:04.519 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:04 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:56:04.520 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:04 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:56:04.520 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:04 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:56:04.520 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:04 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:56:04.570 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:04 smithi183 ceph-mon[27058]: pgmap v285: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.7 MiB/s wr, 817 op/s 2024-04-02T14:56:04.570 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:04 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:56:04.570 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:04 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:56:04.570 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:04 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T14:56:04.570 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:04 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T14:56:05.286 INFO:teuthology.orchestra.run.smithi122.stdout: 5 53641 16.90 MB/sec warmup 91 sec latency 29.313 ms 2024-04-02T14:56:05.786 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:05 smithi122 ceph-mon[24801]: Standby manager daemon smithi183.smxedq restarted 2024-04-02T14:56:05.786 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:05 smithi122 ceph-mon[24801]: Standby manager daemon smithi183.smxedq started 2024-04-02T14:56:05.787 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:05 smithi122 ceph-mon[24801]: pgmap v286: 97 pgs: 97 active+clean; 281 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.1 MiB/s wr, 746 op/s 2024-04-02T14:56:05.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:05 smithi183 ceph-mon[27058]: Standby manager daemon smithi183.smxedq restarted 2024-04-02T14:56:05.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:05 smithi183 ceph-mon[27058]: Standby manager daemon smithi183.smxedq started 2024-04-02T14:56:05.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:05 smithi183 ceph-mon[27058]: pgmap v286: 97 pgs: 97 active+clean; 281 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.1 MiB/s wr, 746 op/s 2024-04-02T14:56:06.286 INFO:teuthology.orchestra.run.smithi122.stdout: 5 54102 16.78 MB/sec warmup 92 sec latency 41.023 ms 2024-04-02T14:56:06.802 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:06 smithi122 ceph-mon[24801]: mgrmap e22: smithi122.szjkfg(active, since 7m), standbys: smithi183.smxedq 2024-04-02T14:56:06.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:06 smithi183 ceph-mon[27058]: mgrmap e22: smithi122.szjkfg(active, since 7m), standbys: smithi183.smxedq 2024-04-02T14:56:07.286 INFO:teuthology.orchestra.run.smithi122.stdout: 5 54542 16.71 MB/sec warmup 93 sec latency 74.689 ms 2024-04-02T14:56:07.585 INFO:teuthology.orchestra.run.smithi122.stdout:true 2024-04-02T14:56:07.827 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:07 smithi122 ceph-mon[24801]: Failed to load ceph-mgr modules: restful, mds_autoscaler, osd_perf_query, snap_schedule, dashboard, zabbix, balancer, influx, alerts, iostat, rgw, rbd_support, progress, pg_autoscaler, devicehealth, k8sevents, rook, diskprediction_local, mirroring, selftest, telegraf, test_orchestrator, localpool, crash, insights, orchestrator, osd_support, volumes, stats, cephadm, telemetry, prometheus, status, nfs 2024-04-02T14:56:07.828 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:07 smithi122 ceph-mon[24801]: pgmap v287: 97 pgs: 97 active+clean; 290 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.2 MiB/s wr, 768 op/s 2024-04-02T14:56:07.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:07 smithi183 ceph-mon[27058]: Failed to load ceph-mgr modules: restful, mds_autoscaler, osd_perf_query, snap_schedule, dashboard, zabbix, balancer, influx, alerts, iostat, rgw, rbd_support, progress, pg_autoscaler, devicehealth, k8sevents, rook, diskprediction_local, mirroring, selftest, telegraf, test_orchestrator, localpool, crash, insights, orchestrator, osd_support, volumes, stats, cephadm, telemetry, prometheus, status, nfs 2024-04-02T14:56:07.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:07 smithi183 ceph-mon[27058]: pgmap v287: 97 pgs: 97 active+clean; 290 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.2 MiB/s wr, 768 op/s 2024-04-02T14:56:08.286 INFO:teuthology.orchestra.run.smithi122.stdout: 5 54981 16.63 MB/sec warmup 94 sec latency 108.303 ms 2024-04-02T14:56:08.306 INFO:teuthology.orchestra.run.smithi122.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T14:56:08.306 INFO:teuthology.orchestra.run.smithi122.stdout:alertmanager.smithi122 smithi122 *:9093,9094 running (6m) 96s ago 8m 24.3M - ba2b418f427c 87ea6de784f7 2024-04-02T14:56:08.306 INFO:teuthology.orchestra.run.smithi122.stdout:crash.smithi122 smithi122 running (8m) 96s ago 8m 6400k - 17.2.0 e1d6a67b021e 140dc3b57d86 2024-04-02T14:56:08.306 INFO:teuthology.orchestra.run.smithi122.stdout:crash.smithi183 smithi183 running (7m) 98s ago 7m 6400k - 17.2.0 e1d6a67b021e 1262330a9cca 2024-04-02T14:56:08.306 INFO:teuthology.orchestra.run.smithi122.stdout:grafana.smithi122 smithi122 *:3000 running (6m) 96s ago 7m 52.6M - 8.3.5 dad864ee21e9 f0623c4719c2 2024-04-02T14:56:08.306 INFO:teuthology.orchestra.run.smithi122.stdout:mds.foofs.smithi122.cghwpx smithi122 running (2m) 96s ago 2m 13.4M - 17.2.0 e1d6a67b021e 5161b53cd4a7 2024-04-02T14:56:08.307 INFO:teuthology.orchestra.run.smithi122.stdout:mds.foofs.smithi183.idnham smithi183 running (2m) 98s ago 2m 14.3M - 17.2.0 e1d6a67b021e 25b5a64c8ee4 2024-04-02T14:56:08.307 INFO:teuthology.orchestra.run.smithi122.stdout:mgr.smithi122.szjkfg smithi122 *:9283 running (9m) 96s ago 9m 459M - 17.2.0 e1d6a67b021e 7583b48bda3e 2024-04-02T14:56:08.309 INFO:teuthology.orchestra.run.smithi122.stdout:mgr.smithi183.smxedq smithi183 *:8443,9283 starting - - - - 2024-04-02T14:56:08.309 INFO:teuthology.orchestra.run.smithi122.stdout:mon.smithi122 smithi122 running (9m) 96s ago 9m 45.0M 2048M 17.2.0 e1d6a67b021e d8f91f766e22 2024-04-02T14:56:08.309 INFO:teuthology.orchestra.run.smithi122.stdout:mon.smithi183 smithi183 running (7m) 98s ago 7m 37.8M 2048M 17.2.0 e1d6a67b021e d6d4734135ac 2024-04-02T14:56:08.309 INFO:teuthology.orchestra.run.smithi122.stdout:nfs.foo.0.0.smithi122.iqmcyp smithi122 *:2049 running (106s) 96s ago 106s 46.9M - 4.0 e1d6a67b021e 15c2856887b6 2024-04-02T14:56:08.309 INFO:teuthology.orchestra.run.smithi122.stdout:nfs.foo.1.0.smithi183.mxktoi smithi183 *:2049 running (101s) 98s ago 101s 46.7M - 4.0 e1d6a67b021e 545c7ba6873c 2024-04-02T14:56:08.309 INFO:teuthology.orchestra.run.smithi122.stdout:node-exporter.smithi122 smithi122 *:9100 running (7m) 96s ago 7m 17.5M - 1dbe0e931976 8815d797e9e2 2024-04-02T14:56:08.309 INFO:teuthology.orchestra.run.smithi122.stdout:node-exporter.smithi183 smithi183 *:9100 running (7m) 98s ago 7m 17.5M - 1dbe0e931976 6c7613782142 2024-04-02T14:56:08.309 INFO:teuthology.orchestra.run.smithi122.stdout:osd.0 smithi122 running (6m) 96s ago 6m 42.1M 3216M 17.2.0 e1d6a67b021e bee30cdc6385 2024-04-02T14:56:08.309 INFO:teuthology.orchestra.run.smithi122.stdout:osd.1 smithi122 running (6m) 96s ago 6m 42.5M 3216M 17.2.0 e1d6a67b021e 92d52898245e 2024-04-02T14:56:08.309 INFO:teuthology.orchestra.run.smithi122.stdout:osd.2 smithi122 running (5m) 96s ago 5m 41.5M 3216M 17.2.0 e1d6a67b021e b88321d239b1 2024-04-02T14:56:08.309 INFO:teuthology.orchestra.run.smithi122.stdout:osd.3 smithi122 running (5m) 96s ago 5m 42.8M 3216M 17.2.0 e1d6a67b021e 1aec61f33b52 2024-04-02T14:56:08.310 INFO:teuthology.orchestra.run.smithi122.stdout:osd.4 smithi183 running (4m) 98s ago 4m 39.8M 3984M 17.2.0 e1d6a67b021e 0d1dbe82a342 2024-04-02T14:56:08.310 INFO:teuthology.orchestra.run.smithi122.stdout:osd.5 smithi183 running (4m) 98s ago 4m 42.7M 3984M 17.2.0 e1d6a67b021e 9bc6c62fa5cb 2024-04-02T14:56:08.310 INFO:teuthology.orchestra.run.smithi122.stdout:osd.6 smithi183 running (3m) 98s ago 3m 39.4M 3984M 17.2.0 e1d6a67b021e f9c8a0a7bfdc 2024-04-02T14:56:08.310 INFO:teuthology.orchestra.run.smithi122.stdout:osd.7 smithi183 running (3m) 98s ago 3m 44.3M 3984M 17.2.0 e1d6a67b021e 1406cf2e45a8 2024-04-02T14:56:08.310 INFO:teuthology.orchestra.run.smithi122.stdout:prometheus.smithi122 smithi122 *:9095 running (6m) 96s ago 6m 61.9M - 514e6a882f6e be604b0d0ed3 2024-04-02T14:56:08.699 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:08 smithi122 ceph-mon[24801]: from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:08.699 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:08 smithi122 ceph-mon[24801]: from='client.14652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:08.699 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:08 smithi122 ceph-mon[24801]: from='client.14656 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:08.771 INFO:teuthology.orchestra.run.smithi122.stdout:{ 2024-04-02T14:56:08.771 INFO:teuthology.orchestra.run.smithi122.stdout: "mon": { 2024-04-02T14:56:08.771 INFO:teuthology.orchestra.run.smithi122.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T14:56:08.771 INFO:teuthology.orchestra.run.smithi122.stdout: }, 2024-04-02T14:56:08.771 INFO:teuthology.orchestra.run.smithi122.stdout: "mgr": { 2024-04-02T14:56:08.771 INFO:teuthology.orchestra.run.smithi122.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-02T14:56:08.771 INFO:teuthology.orchestra.run.smithi122.stdout: "ceph version 19.0.0-2592-g3f3f3ed9 (3f3f3ed992429b85be729743ad3b39d52dd90744) squid (dev)": 1 2024-04-02T14:56:08.771 INFO:teuthology.orchestra.run.smithi122.stdout: }, 2024-04-02T14:56:08.771 INFO:teuthology.orchestra.run.smithi122.stdout: "osd": { 2024-04-02T14:56:08.771 INFO:teuthology.orchestra.run.smithi122.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T14:56:08.771 INFO:teuthology.orchestra.run.smithi122.stdout: }, 2024-04-02T14:56:08.772 INFO:teuthology.orchestra.run.smithi122.stdout: "mds": { 2024-04-02T14:56:08.772 INFO:teuthology.orchestra.run.smithi122.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T14:56:08.772 INFO:teuthology.orchestra.run.smithi122.stdout: }, 2024-04-02T14:56:08.772 INFO:teuthology.orchestra.run.smithi122.stdout: "overall": { 2024-04-02T14:56:08.772 INFO:teuthology.orchestra.run.smithi122.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-02T14:56:08.772 INFO:teuthology.orchestra.run.smithi122.stdout: "ceph version 19.0.0-2592-g3f3f3ed9 (3f3f3ed992429b85be729743ad3b39d52dd90744) squid (dev)": 1 2024-04-02T14:56:08.772 INFO:teuthology.orchestra.run.smithi122.stdout: } 2024-04-02T14:56:08.772 INFO:teuthology.orchestra.run.smithi122.stdout:} 2024-04-02T14:56:08.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:08 smithi183 ceph-mon[27058]: from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:08.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:08 smithi183 ceph-mon[27058]: from='client.14652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:08.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:08 smithi183 ceph-mon[27058]: from='client.14656 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:09.182 INFO:teuthology.orchestra.run.smithi122.stdout:{ 2024-04-02T14:56:09.183 INFO:teuthology.orchestra.run.smithi122.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac", 2024-04-02T14:56:09.183 INFO:teuthology.orchestra.run.smithi122.stdout: "in_progress": true, 2024-04-02T14:56:09.183 INFO:teuthology.orchestra.run.smithi122.stdout: "services_complete": [], 2024-04-02T14:56:09.183 INFO:teuthology.orchestra.run.smithi122.stdout: "progress": "", 2024-04-02T14:56:09.183 INFO:teuthology.orchestra.run.smithi122.stdout: "message": "Currently upgrading mgr daemons" 2024-04-02T14:56:09.183 INFO:teuthology.orchestra.run.smithi122.stdout:} 2024-04-02T14:56:09.286 INFO:teuthology.orchestra.run.smithi122.stdout: 5 55457 16.54 MB/sec warmup 95 sec latency 309.383 ms 2024-04-02T14:56:09.679 INFO:teuthology.orchestra.run.smithi122.stdout:HEALTH_OK 2024-04-02T14:56:09.721 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:09 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/1858285038' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T14:56:09.721 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:09 smithi122 ceph-mon[24801]: from='client.14664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:09.722 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:09 smithi122 ceph-mon[24801]: pgmap v288: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.8 MiB/s wr, 974 op/s 2024-04-02T14:56:09.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:09 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/1858285038' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T14:56:09.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:09 smithi183 ceph-mon[27058]: from='client.14664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T14:56:09.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:09 smithi183 ceph-mon[27058]: pgmap v288: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.8 MiB/s wr, 974 op/s 2024-04-02T14:56:10.286 INFO:teuthology.orchestra.run.smithi122.stdout: 5 55934 16.52 MB/sec warmup 96 sec latency 67.241 ms 2024-04-02T14:56:10.726 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:10 smithi122 ceph-mon[24801]: from='client.? 172.21.15.122:0/2434613979' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T14:56:10.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:10 smithi183 ceph-mon[27058]: from='client.? 172.21.15.122:0/2434613979' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T14:56:11.287 INFO:teuthology.orchestra.run.smithi122.stdout: 5 56526 16.63 MB/sec warmup 97 sec latency 65.010 ms 2024-04-02T14:56:11.760 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:11 smithi122 ceph-mon[24801]: pgmap v289: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.5 MiB/s wr, 811 op/s 2024-04-02T14:56:11.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:11 smithi183 ceph-mon[27058]: pgmap v289: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.5 MiB/s wr, 811 op/s 2024-04-02T14:56:12.286 INFO:teuthology.orchestra.run.smithi122.stdout: 5 56960 16.63 MB/sec warmup 98 sec latency 38.486 ms 2024-04-02T14:56:13.287 INFO:teuthology.orchestra.run.smithi122.stdout: 5 57287 16.51 MB/sec warmup 99 sec latency 32.224 ms 2024-04-02T14:56:14.287 INFO:teuthology.orchestra.run.smithi122.stdout: 5 57566 16.37 MB/sec warmup 100 sec latency 72.183 ms 2024-04-02T14:56:14.318 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:14 smithi122 ceph-mon[24801]: pgmap v290: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 864 op/s 2024-04-02T14:56:14.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:14 smithi183 ceph-mon[27058]: pgmap v290: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 864 op/s 2024-04-02T14:56:15.287 INFO:teuthology.orchestra.run.smithi122.stdout: 5 58084 16.34 MB/sec warmup 101 sec latency 48.060 ms 2024-04-02T14:56:15.828 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:15 smithi122 ceph-mon[24801]: pgmap v291: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 734 op/s 2024-04-02T14:56:15.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:15 smithi183 ceph-mon[27058]: pgmap v291: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 734 op/s 2024-04-02T14:56:16.287 INFO:teuthology.orchestra.run.smithi122.stdout: 5 58904 16.32 MB/sec warmup 102 sec latency 26.143 ms 2024-04-02T14:56:17.287 INFO:teuthology.orchestra.run.smithi122.stdout: 5 59746 16.45 MB/sec warmup 103 sec latency 21.027 ms 2024-04-02T14:56:18.287 INFO:teuthology.orchestra.run.smithi122.stdout: 5 60581 16.61 MB/sec warmup 104 sec latency 47.535 ms 2024-04-02T14:56:18.344 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:18 smithi122 ceph-mon[24801]: pgmap v292: 97 pgs: 97 active+clean; 309 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.7 MiB/s wr, 777 op/s 2024-04-02T14:56:18.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:18 smithi183 ceph-mon[27058]: pgmap v292: 97 pgs: 97 active+clean; 309 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.7 MiB/s wr, 777 op/s 2024-04-02T14:56:19.287 INFO:teuthology.orchestra.run.smithi122.stdout: 5 61102 16.51 MB/sec warmup 105 sec latency 66.573 ms 2024-04-02T14:56:20.287 INFO:teuthology.orchestra.run.smithi122.stdout: 5 61775 16.51 MB/sec warmup 106 sec latency 54.905 ms 2024-04-02T14:56:20.363 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:20 smithi122 ceph-mon[24801]: pgmap v293: 97 pgs: 97 active+clean; 314 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.6 MiB/s wr, 854 op/s 2024-04-02T14:56:20.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:20 smithi183 ceph-mon[27058]: pgmap v293: 97 pgs: 97 active+clean; 314 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.6 MiB/s wr, 854 op/s 2024-04-02T14:56:21.287 INFO:teuthology.orchestra.run.smithi122.stdout: 5 62515 16.46 MB/sec warmup 107 sec latency 51.299 ms 2024-04-02T14:56:22.293 INFO:teuthology.orchestra.run.smithi122.stdout: 5 63156 16.51 MB/sec warmup 108 sec latency 29.335 ms 2024-04-02T14:56:22.379 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:22 smithi122 ceph-mon[24801]: pgmap v294: 97 pgs: 97 active+clean; 315 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.4 MiB/s wr, 663 op/s 2024-04-02T14:56:22.379 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:22 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:56:22.380 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:22 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:56:22.650 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:22 smithi183 ceph-mon[27058]: pgmap v294: 97 pgs: 97 active+clean; 315 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.4 MiB/s wr, 663 op/s 2024-04-02T14:56:22.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:22 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:56:22.651 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:22 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:56:23.293 INFO:teuthology.orchestra.run.smithi122.stdout: 5 63861 16.63 MB/sec warmup 109 sec latency 22.858 ms 2024-04-02T14:56:24.293 INFO:teuthology.orchestra.run.smithi122.stdout: 5 64378 16.60 MB/sec warmup 110 sec latency 29.707 ms 2024-04-02T14:56:24.400 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:24 smithi122 ceph-mon[24801]: pgmap v295: 97 pgs: 97 active+clean; 326 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 9.1 MiB/s wr, 931 op/s 2024-04-02T14:56:24.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:24 smithi183 ceph-mon[27058]: pgmap v295: 97 pgs: 97 active+clean; 326 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 9.1 MiB/s wr, 931 op/s 2024-04-02T14:56:25.293 INFO:teuthology.orchestra.run.smithi122.stdout: 5 64921 16.53 MB/sec warmup 111 sec latency 21.168 ms 2024-04-02T14:56:25.656 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:25 smithi122 ceph-mon[24801]: pgmap v296: 97 pgs: 97 active+clean; 325 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 8.1 MiB/s wr, 750 op/s 2024-04-02T14:56:25.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:25 smithi183 ceph-mon[27058]: pgmap v296: 97 pgs: 97 active+clean; 325 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 8.1 MiB/s wr, 750 op/s 2024-04-02T14:56:26.293 INFO:teuthology.orchestra.run.smithi122.stdout: 5 65595 16.52 MB/sec warmup 112 sec latency 85.640 ms 2024-04-02T14:56:27.293 INFO:teuthology.orchestra.run.smithi122.stdout: 5 66294 16.49 MB/sec warmup 113 sec latency 36.026 ms 2024-04-02T14:56:27.719 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:27 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:56:27.720 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:27 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:56:27.720 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:27 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:56:27.720 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:27 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:56:27.720 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:27 smithi122 ceph-mon[24801]: Upgrade: Need to upgrade myself (mgr.smithi122.szjkfg) 2024-04-02T14:56:27.720 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:27 smithi122 ceph-mon[24801]: Failing over to other MGR 2024-04-02T14:56:27.720 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:27 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mgr fail", "who": "smithi122.szjkfg"}]: dispatch 2024-04-02T14:56:27.720 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:27 smithi122 ceph-mon[24801]: osdmap e71: 8 total, 8 up, 8 in 2024-04-02T14:56:27.720 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:27 smithi122 ceph-mon[24801]: pgmap v298: 97 pgs: 97 active+clean; 327 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 8.9 MiB/s wr, 803 op/s 2024-04-02T14:56:27.720 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:27 smithi122 ceph-mon[24801]: Standby manager daemon smithi122.szjkfg started 2024-04-02T14:56:27.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:27 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:56:27.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:27 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:56:27.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:27 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' 2024-04-02T14:56:27.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:27 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T14:56:27.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:27 smithi183 ceph-mon[27058]: Upgrade: Need to upgrade myself (mgr.smithi122.szjkfg) 2024-04-02T14:56:27.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:27 smithi183 ceph-mon[27058]: Failing over to other MGR 2024-04-02T14:56:27.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:27 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "mgr fail", "who": "smithi122.szjkfg"}]: dispatch 2024-04-02T14:56:27.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:27 smithi183 ceph-mon[27058]: osdmap e71: 8 total, 8 up, 8 in 2024-04-02T14:56:27.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:27 smithi183 ceph-mon[27058]: pgmap v298: 97 pgs: 97 active+clean; 327 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 8.9 MiB/s wr, 803 op/s 2024-04-02T14:56:27.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:27 smithi183 ceph-mon[27058]: Standby manager daemon smithi122.szjkfg started 2024-04-02T14:56:28.293 INFO:teuthology.orchestra.run.smithi122.stdout: 5 66841 16.54 MB/sec warmup 114 sec latency 27.408 ms 2024-04-02T14:56:28.762 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "mon metadata", "id": "smithi122"}]: dispatch 2024-04-02T14:56:28.762 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "mon metadata", "id": "smithi183"}]: dispatch 2024-04-02T14:56:28.762 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi122.cghwpx"}]: dispatch 2024-04-02T14:56:28.762 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi183.idnham"}]: dispatch 2024-04-02T14:56:28.762 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "mgr metadata", "who": "smithi183.smxedq", "id": "smithi183.smxedq"}]: dispatch 2024-04-02T14:56:28.762 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "mgr metadata", "who": "smithi122.szjkfg", "id": "smithi122.szjkfg"}]: dispatch 2024-04-02T14:56:28.762 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:56:28.762 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:56:28.762 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:56:28.762 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:56:28.762 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:56:28.762 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:56:28.762 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T14:56:28.763 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "mgr fail", "who": "smithi122.szjkfg"}]': finished 2024-04-02T14:56:28.763 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T14:56:28.763 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: mgrmap e23: smithi183.smxedq(active, starting, since 0.995352s), standbys: smithi122.szjkfg 2024-04-02T14:56:28.763 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T14:56:28.763 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T14:56:28.763 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T14:56:28.763 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:28 smithi122 ceph-mon[24801]: Manager daemon smithi183.smxedq is now available 2024-04-02T14:56:28.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "mon metadata", "id": "smithi122"}]: dispatch 2024-04-02T14:56:28.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "mon metadata", "id": "smithi183"}]: dispatch 2024-04-02T14:56:28.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi122.cghwpx"}]: dispatch 2024-04-02T14:56:28.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi183.idnham"}]: dispatch 2024-04-02T14:56:28.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "mgr metadata", "who": "smithi183.smxedq", "id": "smithi183.smxedq"}]: dispatch 2024-04-02T14:56:28.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "mgr metadata", "who": "smithi122.szjkfg", "id": "smithi122.szjkfg"}]: dispatch 2024-04-02T14:56:28.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T14:56:28.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T14:56:28.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T14:56:28.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T14:56:28.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T14:56:28.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T14:56:28.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T14:56:28.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: from='mgr.14200 172.21.15.122:0/2248692041' entity='mgr.smithi122.szjkfg' cmd='[{"prefix": "mgr fail", "who": "smithi122.szjkfg"}]': finished 2024-04-02T14:56:28.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T14:56:28.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: mgrmap e23: smithi183.smxedq(active, starting, since 0.995352s), standbys: smithi122.szjkfg 2024-04-02T14:56:28.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T14:56:28.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T14:56:28.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: from='mgr.24389 172.21.15.183:0/220885637' entity='mgr.smithi183.smxedq' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T14:56:28.900 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:28 smithi183 ceph-mon[27058]: Manager daemon smithi183.smxedq is now available 2024-04-02T14:56:29.293 INFO:teuthology.orchestra.run.smithi122.stdout: 5 67399 16.59 MB/sec warmup 115 sec latency 86.486 ms 2024-04-02T14:56:29.769 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:29 smithi122 ceph-mon[24801]: mgrmap e24: smithi183.smxedq(active, since 2s), standbys: smithi122.szjkfg 2024-04-02T14:56:29.769 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:29 smithi122 ceph-mon[24801]: pgmap v3: 97 pgs: 97 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-02T14:56:29.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:29 smithi183 ceph-mon[27058]: mgrmap e24: smithi183.smxedq(active, since 2s), standbys: smithi122.szjkfg 2024-04-02T14:56:29.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:29 smithi183 ceph-mon[27058]: pgmap v3: 97 pgs: 97 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-02T14:56:30.293 INFO:teuthology.orchestra.run.smithi122.stdout: 5 67825 16.56 MB/sec warmup 116 sec latency 30.687 ms 2024-04-02T14:56:30.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:30 smithi183 ceph-mon[27058]: pgmap v4: 97 pgs: 97 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-02T14:56:30.648 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:30 smithi183 ceph-mon[27058]: Health check failed: 16 mgr modules have failed dependencies (MGR_MODULE_DEPENDENCY) 2024-04-02T14:56:30.774 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:30 smithi122 ceph-mon[24801]: pgmap v4: 97 pgs: 97 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-02T14:56:30.774 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:30 smithi122 ceph-mon[24801]: Health check failed: 16 mgr modules have failed dependencies (MGR_MODULE_DEPENDENCY) 2024-04-02T14:56:31.293 INFO:teuthology.orchestra.run.smithi122.stdout: 5 68322 16.46 MB/sec warmup 117 sec latency 24.066 ms 2024-04-02T14:56:32.293 INFO:teuthology.orchestra.run.smithi122.stdout: 5 68959 16.45 MB/sec warmup 118 sec latency 28.468 ms 2024-04-02T14:56:32.782 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:32 smithi122 ceph-mon[24801]: pgmap v5: 97 pgs: 97 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-02T14:56:32.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:32 smithi183 ceph-mon[27058]: pgmap v5: 97 pgs: 97 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-02T14:56:33.293 INFO:teuthology.orchestra.run.smithi122.stdout: 5 69616 16.41 MB/sec warmup 119 sec latency 29.928 ms 2024-04-02T14:56:34.805 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:34 smithi122 ceph-mon[24801]: pgmap v6: 97 pgs: 97 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-02T14:56:34.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:34 smithi183 ceph-mon[27058]: pgmap v6: 97 pgs: 97 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-02T14:56:35.293 INFO:teuthology.orchestra.run.smithi122.stdout: 5 70762 27.00 MB/sec execute 1 sec latency 43.659 ms 2024-04-02T14:56:36.293 INFO:teuthology.orchestra.run.smithi122.stdout: 5 71351 22.90 MB/sec execute 2 sec latency 18.620 ms 2024-04-02T14:56:36.802 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:36 smithi122 ceph-mon[24801]: pgmap v7: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.3 MiB/s wr, 680 op/s 2024-04-02T14:56:36.803 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:36 smithi122 ceph-mon[24801]: Standby manager daemon smithi122.szjkfg restarted 2024-04-02T14:56:36.803 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:36 smithi122 ceph-mon[24801]: Standby manager daemon smithi122.szjkfg started 2024-04-02T14:56:36.803 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:36 smithi122 ceph-mon[24801]: from='mgr.? 172.21.15.122:0/2925907027' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi122.szjkfg/crt"}]: dispatch 2024-04-02T14:56:36.803 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:36 smithi122 ceph-mon[24801]: from='mgr.? 172.21.15.122:0/2925907027' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T14:56:36.803 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:36 smithi122 ceph-mon[24801]: from='mgr.? 172.21.15.122:0/2925907027' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi122.szjkfg/key"}]: dispatch 2024-04-02T14:56:36.803 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:36 smithi122 ceph-mon[24801]: from='mgr.? 172.21.15.122:0/2925907027' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T14:56:36.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:36 smithi183 ceph-mon[27058]: pgmap v7: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.3 MiB/s wr, 680 op/s 2024-04-02T14:56:36.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:36 smithi183 ceph-mon[27058]: Standby manager daemon smithi122.szjkfg restarted 2024-04-02T14:56:36.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:36 smithi183 ceph-mon[27058]: Standby manager daemon smithi122.szjkfg started 2024-04-02T14:56:36.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:36 smithi183 ceph-mon[27058]: from='mgr.? 172.21.15.122:0/2925907027' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi122.szjkfg/crt"}]: dispatch 2024-04-02T14:56:36.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:36 smithi183 ceph-mon[27058]: from='mgr.? 172.21.15.122:0/2925907027' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T14:56:36.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:36 smithi183 ceph-mon[27058]: from='mgr.? 172.21.15.122:0/2925907027' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi122.szjkfg/key"}]: dispatch 2024-04-02T14:56:36.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:36 smithi183 ceph-mon[27058]: from='mgr.? 172.21.15.122:0/2925907027' entity='mgr.smithi122.szjkfg' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T14:56:37.293 INFO:teuthology.orchestra.run.smithi122.stdout: 5 71887 17.26 MB/sec execute 3 sec latency 22.556 ms 2024-04-02T14:56:37.833 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:37 smithi122 ceph-mon[24801]: mgrmap e25: smithi183.smxedq(active, since 10s), standbys: smithi122.szjkfg 2024-04-02T14:56:37.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:37 smithi183 ceph-mon[27058]: mgrmap e25: smithi183.smxedq(active, since 10s), standbys: smithi122.szjkfg 2024-04-02T14:56:38.293 INFO:teuthology.orchestra.run.smithi122.stdout: 5 72491 15.99 MB/sec execute 4 sec latency 26.336 ms 2024-04-02T14:56:38.849 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:38 smithi122 ceph-mon[24801]: pgmap v8: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.2 MiB/s wr, 529 op/s 2024-04-02T14:56:38.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:38 smithi183 ceph-mon[27058]: pgmap v8: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.2 MiB/s wr, 529 op/s 2024-04-02T14:56:39.293 INFO:teuthology.orchestra.run.smithi122.stdout: 5 73086 15.58 MB/sec execute 5 sec latency 37.932 ms 2024-04-02T14:56:40.026 INFO:teuthology.orchestra.run.smithi122.stderr:no valid command found; 10 closest matches: 2024-04-02T14:56:40.026 INFO:teuthology.orchestra.run.smithi122.stderr:pg stat 2024-04-02T14:56:40.027 INFO:teuthology.orchestra.run.smithi122.stderr:pg getmap 2024-04-02T14:56:40.027 INFO:teuthology.orchestra.run.smithi122.stderr:pg dump [...] 2024-04-02T14:56:40.027 INFO:teuthology.orchestra.run.smithi122.stderr:pg dump_json [...] 2024-04-02T14:56:40.027 INFO:teuthology.orchestra.run.smithi122.stderr:pg dump_pools_json 2024-04-02T14:56:40.027 INFO:teuthology.orchestra.run.smithi122.stderr:pg ls-by-pool [...] 2024-04-02T14:56:40.027 INFO:teuthology.orchestra.run.smithi122.stderr:pg ls-by-primary [] [...] 2024-04-02T14:56:40.027 INFO:teuthology.orchestra.run.smithi122.stderr:pg ls-by-osd [] [...] 2024-04-02T14:56:40.027 INFO:teuthology.orchestra.run.smithi122.stderr:pg ls [] [...] 2024-04-02T14:56:40.027 INFO:teuthology.orchestra.run.smithi122.stderr:pg dump_stuck [...] [] 2024-04-02T14:56:40.027 INFO:teuthology.orchestra.run.smithi122.stderr:Error EINVAL: invalid command 2024-04-02T14:56:40.293 INFO:teuthology.orchestra.run.smithi122.stdout: 5 73730 15.99 MB/sec execute 6 sec latency 31.398 ms 2024-04-02T14:56:40.868 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:40 smithi122 ceph-mon[24801]: pgmap v9: 97 pgs: 97 active+clean; 344 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 808 op/s 2024-04-02T14:56:40.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:40 smithi183 ceph-mon[27058]: pgmap v9: 97 pgs: 97 active+clean; 344 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 808 op/s 2024-04-02T14:56:41.226 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch ps' 2024-04-02T14:56:41.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 74560 18.71 MB/sec execute 7 sec latency 21.798 ms 2024-04-02T14:56:42.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 75008 17.76 MB/sec execute 8 sec latency 76.493 ms 2024-04-02T14:56:42.885 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:42 smithi122 ceph-mon[24801]: pgmap v10: 97 pgs: 97 active+clean; 344 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.9 MiB/s wr, 740 op/s 2024-04-02T14:56:42.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:42 smithi183 ceph-mon[27058]: pgmap v10: 97 pgs: 97 active+clean; 344 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.9 MiB/s wr, 740 op/s 2024-04-02T14:56:43.293 INFO:teuthology.orchestra.run.smithi122.stdout: 5 75383 16.28 MB/sec execute 9 sec latency 35.891 ms 2024-04-02T14:56:44.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 75974 15.93 MB/sec execute 10 sec latency 30.211 ms 2024-04-02T14:56:44.648 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:44 smithi122 ceph-mon[24801]: pgmap v11: 97 pgs: 97 active+clean; 344 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.9 MiB/s wr, 740 op/s 2024-04-02T14:56:44.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:44 smithi183 ceph-mon[27058]: pgmap v11: 97 pgs: 97 active+clean; 344 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.9 MiB/s wr, 740 op/s 2024-04-02T14:56:44.911 INFO:teuthology.orchestra.run.smithi122.stderr:no valid command found; 10 closest matches: 2024-04-02T14:56:44.912 INFO:teuthology.orchestra.run.smithi122.stderr:pg stat 2024-04-02T14:56:44.912 INFO:teuthology.orchestra.run.smithi122.stderr:pg getmap 2024-04-02T14:56:44.912 INFO:teuthology.orchestra.run.smithi122.stderr:pg dump [...] 2024-04-02T14:56:44.912 INFO:teuthology.orchestra.run.smithi122.stderr:pg dump_json [...] 2024-04-02T14:56:44.912 INFO:teuthology.orchestra.run.smithi122.stderr:pg dump_pools_json 2024-04-02T14:56:44.912 INFO:teuthology.orchestra.run.smithi122.stderr:pg ls-by-pool [...] 2024-04-02T14:56:44.912 INFO:teuthology.orchestra.run.smithi122.stderr:pg ls-by-primary [] [...] 2024-04-02T14:56:44.912 INFO:teuthology.orchestra.run.smithi122.stderr:pg ls-by-osd [] [...] 2024-04-02T14:56:44.912 INFO:teuthology.orchestra.run.smithi122.stderr:pg ls [] [...] 2024-04-02T14:56:44.912 INFO:teuthology.orchestra.run.smithi122.stderr:pg dump_stuck [...] [] 2024-04-02T14:56:44.912 INFO:teuthology.orchestra.run.smithi122.stderr:Error EINVAL: invalid command 2024-04-02T14:56:45.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 76820 16.14 MB/sec execute 11 sec latency 20.796 ms 2024-04-02T14:56:46.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 77580 17.08 MB/sec execute 12 sec latency 25.557 ms 2024-04-02T14:56:46.421 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-04-02T14:56:46.422 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa/tasks/cephadm.py", line 1459, in shell _shell( File "/home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa/tasks/cephadm.py", line 110, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi122 with status 22: "sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch ps'" 2024-04-02T14:56:46.627 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=7c9369ba58ea4da69557a7243ac5e8bf Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa/tasks/cephadm.py", line 1459, in shell _shell( File "/home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa/tasks/cephadm.py", line 110, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi122 with status 22: "sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch ps'" 2024-04-02T14:56:46.630 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-02T14:56:46.640 INFO:tasks.cephadm:Teardown begin 2024-04-02T14:56:46.641 DEBUG:teuthology.orchestra.run.smithi122:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-02T14:56:46.646 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:46 smithi122 ceph-mon[24801]: pgmap v12: 97 pgs: 97 active+clean; 351 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.06k op/s 2024-04-02T14:56:46.685 DEBUG:teuthology.orchestra.run.smithi183:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-02T14:56:46.720 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-02T14:56:46.720 DEBUG:teuthology.orchestra.run.smithi122:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-02T14:56:46.744 DEBUG:teuthology.orchestra.run.smithi183:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-02T14:56:46.776 INFO:tasks.cephadm:Stopping all daemons... 2024-04-02T14:56:46.776 INFO:tasks.cephadm.mon.smithi122:Stopping mon.smithi122... 2024-04-02T14:56:46.777 DEBUG:teuthology.orchestra.run.smithi122:> sudo systemctl stop ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@mon.smithi122 2024-04-02T14:56:46.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:46 smithi183 ceph-mon[27058]: pgmap v12: 97 pgs: 97 active+clean; 351 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.06k op/s 2024-04-02T14:56:47.084 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:46 smithi122 systemd[1]: Stopping Ceph mon.smithi122 for bc643104-f0ff-11ee-b647-cb9ed24678a4... 2024-04-02T14:56:47.085 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:46 smithi122 bash[57093]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon.smithi122" found: no such container 2024-04-02T14:56:47.085 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:47 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi122[24797]: 2024-04-02T14:56:47.030+0000 7f800afd2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi122 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T14:56:47.085 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:47 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi122[24797]: 2024-04-02T14:56:47.030+0000 7f800afd2700 -1 mon.smithi122@0(leader) e2 *** Got Signal Terminated *** 2024-04-02T14:56:47.085 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:47 smithi122 podman[57105]: 2024-04-02 14:56:47.051448654 +0000 UTC m=+0.147086768 container died d8f91f766e220cb85c34a154aa046e9156ff779b665feb00ddfedd960176f3e9 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi122, GIT_BRANCH=HEAD, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, version=8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, architecture=x86_64, vcs-type=git, RELEASE=HEAD, name=centos-stream, io.buildah.version=1.19.8) 2024-04-02T14:56:47.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78174 17.66 MB/sec execute 13 sec latency 30.878 ms 2024-04-02T14:56:47.839 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:47 smithi122 podman[57105]: 2024-04-02 14:56:47.703174396 +0000 UTC m=+0.798812495 container cleanup d8f91f766e220cb85c34a154aa046e9156ff779b665feb00ddfedd960176f3e9 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi122, distribution-scope=public, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, vcs-type=git, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, release=754, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, name=centos-stream, ceph=True, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux ) 2024-04-02T14:56:47.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:47 smithi122 bash[57105]: ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi122 2024-04-02T14:56:47.840 INFO:journalctl@ceph.mon.smithi122.smithi122.stdout:Apr 02 14:56:47 smithi122 bash[57132]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon.smithi122" found: no such container 2024-04-02T14:56:48.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78680 17.09 MB/sec execute 14 sec latency 30.932 ms 2024-04-02T14:56:48.481 DEBUG:teuthology.orchestra.run.smithi122:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@mon.smithi122.service' 2024-04-02T14:56:48.572 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T14:56:48.573 INFO:tasks.cephadm.mon.smithi122:Stopped mon.smithi122 2024-04-02T14:56:48.573 INFO:tasks.cephadm.mon.smithi183:Stopping mon.smithi183... 2024-04-02T14:56:48.573 DEBUG:teuthology.orchestra.run.smithi183:> sudo systemctl stop ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@mon.smithi183 2024-04-02T14:56:48.898 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:48 smithi183 systemd[1]: Stopping Ceph mon.smithi183 for bc643104-f0ff-11ee-b647-cb9ed24678a4... 2024-04-02T14:56:48.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:48 smithi183 bash[44335]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon.smithi183" found: no such container 2024-04-02T14:56:48.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:48 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi183[27053]: 2024-04-02T14:56:48.805+0000 7fcb1f173700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi183 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T14:56:48.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:48 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi183[27053]: 2024-04-02T14:56:48.805+0000 7fcb1f173700 -1 mon.smithi183@1(peon) e2 *** Got Signal Terminated *** 2024-04-02T14:56:48.899 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:48 smithi183 podman[44348]: 2024-04-02 14:56:48.847273184 +0000 UTC m=+0.184857782 container died d6d4734135ac9da3330bca54e4b79ac57931a7156f72013f6655de50edf651dc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi183, io.buildah.version=1.19.8, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., architecture=x86_64, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, name=centos-stream, ceph=True, release=754, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-02T14:56:49.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 16.19 MB/sec execute 15 sec latency 496.103 ms 2024-04-02T14:56:49.318 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:49 smithi183 podman[44348]: 2024-04-02 14:56:49.30620498 +0000 UTC m=+0.643789577 container cleanup d6d4734135ac9da3330bca54e4b79ac57931a7156f72013f6655de50edf651dc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi183, build-date=2022-05-03T08:36:31.336870, ceph=True, distribution-scope=public, io.openshift.tags=base centos centos-stream, release=754, name=centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, version=8, architecture=x86_64) 2024-04-02T14:56:49.319 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:49 smithi183 bash[44348]: ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi183 2024-04-02T14:56:49.319 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:49 smithi183 bash[44375]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon.smithi183" found: no such container 2024-04-02T14:56:49.560 DEBUG:teuthology.orchestra.run.smithi183:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@mon.smithi183.service' 2024-04-02T14:56:49.585 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:49 smithi183 podman[44362]: 2024-04-02 14:56:49.566212311 +0000 UTC m=+0.719574796 container remove d6d4734135ac9da3330bca54e4b79ac57931a7156f72013f6655de50edf651dc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-mon-smithi183, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, vcs-type=git, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, ceph=True, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, GIT_CLEAN=True, architecture=x86_64, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD) 2024-04-02T14:56:49.586 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:49 smithi183 systemd[1]: ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@mon.smithi183.service: Deactivated successfully. 2024-04-02T14:56:49.586 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:49 smithi183 systemd[1]: Stopped Ceph mon.smithi183 for bc643104-f0ff-11ee-b647-cb9ed24678a4. 2024-04-02T14:56:49.586 INFO:journalctl@ceph.mon.smithi183.smithi183.stdout:Apr 02 14:56:49 smithi183 systemd[1]: ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@mon.smithi183.service: Consumed 4.276s CPU time. 2024-04-02T14:56:49.613 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T14:56:49.614 INFO:tasks.cephadm.mon.smithi183:Stopped mon.smithi183 2024-04-02T14:56:49.614 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-02T14:56:49.614 DEBUG:teuthology.orchestra.run.smithi122:> sudo systemctl stop ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.0 2024-04-02T14:56:49.912 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:56:49 smithi122 systemd[1]: Stopping Ceph osd.0 for bc643104-f0ff-11ee-b647-cb9ed24678a4... 2024-04-02T14:56:49.913 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:56:49 smithi122 bash[57180]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd.0" found: no such container 2024-04-02T14:56:50.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 15.18 MB/sec execute 16 sec latency 1496.159 ms 2024-04-02T14:56:50.337 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:56:49 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-0[35142]: 2024-04-02T14:56:49.922+0000 7f15fa191700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T14:56:50.337 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:56:49 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-0[35142]: 2024-04-02T14:56:49.922+0000 7f15fa191700 -1 osd.0 71 *** Got signal Terminated *** 2024-04-02T14:56:50.337 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:56:49 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-0[35142]: 2024-04-02T14:56:49.922+0000 7f15fa191700 -1 osd.0 71 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T14:56:51.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 14.29 MB/sec execute 17 sec latency 2496.216 ms 2024-04-02T14:56:52.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 13.49 MB/sec execute 18 sec latency 3496.275 ms 2024-04-02T14:56:53.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 12.78 MB/sec execute 19 sec latency 4496.336 ms 2024-04-02T14:56:54.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 12.14 MB/sec execute 20 sec latency 5496.368 ms 2024-04-02T14:56:55.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 11.57 MB/sec execute 21 sec latency 6496.396 ms 2024-04-02T14:56:55.837 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:56:55 smithi122 podman[57193]: 2024-04-02 14:56:55.585115906 +0000 UTC m=+5.866803643 container died bee30cdc6385e4a063ce25e4a85c666241849ebc85975728906c8e29bc2b13c9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-0, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, ceph=True, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.buildah.version=1.19.8, io.openshift.expose-services=, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-02T14:56:56.207 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:56:55 smithi122 podman[57193]: 2024-04-02 14:56:55.965897449 +0000 UTC m=+6.247585170 container cleanup bee30cdc6385e4a063ce25e4a85c666241849ebc85975728906c8e29bc2b13c9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, GIT_CLEAN=True, maintainer=Guillaume Abrioux , architecture=x86_64, ceph=True, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, release=754, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, vcs-type=git, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public) 2024-04-02T14:56:56.207 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:56:55 smithi122 bash[57193]: ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-0 2024-04-02T14:56:56.207 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:56:55 smithi122 bash[57223]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd.0" found: no such container 2024-04-02T14:56:56.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 11.04 MB/sec execute 22 sec latency 7496.422 ms 2024-04-02T14:56:56.587 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:56:56 smithi122 podman[57210]: 2024-04-02 14:56:56.217304285 +0000 UTC m=+0.636005787 container remove bee30cdc6385e4a063ce25e4a85c666241849ebc85975728906c8e29bc2b13c9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, GIT_BRANCH=HEAD, vcs-type=git, io.buildah.version=1.19.8, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, GIT_CLEAN=True, io.openshift.expose-services=, com.redhat.component=centos-stream-container) 2024-04-02T14:56:56.587 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:56:56 smithi122 podman[57285]: 2024-04-02 14:56:56.3183929 +0000 UTC m=+0.019748412 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T14:56:56.587 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:56:56 smithi122 podman[57285]: 2024-04-02 14:56:56.451572139 +0000 UTC m=+0.152927635 container create 3d177e45cd30e1b69afed5df8f89d1886c3bc09cc40b28171d9ad2a19677381b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-0-deactivate, io.buildah.version=1.19.8, ceph=True, version=8, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, GIT_CLEAN=True, RELEASE=HEAD, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, vcs-type=git, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-02T14:56:57.075 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:56:56 smithi122 podman[57285]: 2024-04-02 14:56:56.721253453 +0000 UTC m=+0.422608947 container init 3d177e45cd30e1b69afed5df8f89d1886c3bc09cc40b28171d9ad2a19677381b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-0-deactivate, vcs-type=git, RELEASE=HEAD, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, ceph=True, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, distribution-scope=public, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-02T14:56:57.075 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:56:56 smithi122 podman[57285]: 2024-04-02 14:56:56.727636899 +0000 UTC m=+0.428992395 container start 3d177e45cd30e1b69afed5df8f89d1886c3bc09cc40b28171d9ad2a19677381b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-0-deactivate, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, vcs-type=git, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, ceph=True, RELEASE=HEAD, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-02T14:56:57.076 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:56:56 smithi122 podman[57285]: 2024-04-02 14:56:56.812977359 +0000 UTC m=+0.514332856 container attach 3d177e45cd30e1b69afed5df8f89d1886c3bc09cc40b28171d9ad2a19677381b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-0-deactivate, GIT_CLEAN=True, io.openshift.expose-services=, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_BRANCH=HEAD, name=centos-stream, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8) 2024-04-02T14:56:57.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 10.56 MB/sec execute 23 sec latency 8496.455 ms 2024-04-02T14:56:57.337 INFO:journalctl@ceph.osd.0.smithi122.stdout:Apr 02 14:56:57 smithi122 podman[57285]: 2024-04-02 14:56:57.08610046 +0000 UTC m=+0.787455962 container died 3d177e45cd30e1b69afed5df8f89d1886c3bc09cc40b28171d9ad2a19677381b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-0-deactivate, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, architecture=x86_64, vendor=Red Hat, Inc., GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, vcs-type=git, ceph=True, version=8, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, name=centos-stream, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-02T14:56:57.753 DEBUG:teuthology.orchestra.run.smithi122:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.0.service' 2024-04-02T14:56:57.792 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T14:56:57.792 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-02T14:56:57.793 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-02T14:56:57.793 DEBUG:teuthology.orchestra.run.smithi122:> sudo systemctl stop ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.1 2024-04-02T14:56:58.047 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:56:57 smithi122 systemd[1]: Stopping Ceph osd.1 for bc643104-f0ff-11ee-b647-cb9ed24678a4... 2024-04-02T14:56:58.047 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:56:57 smithi122 bash[57363]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd.1" found: no such container 2024-04-02T14:56:58.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 10.12 MB/sec execute 24 sec latency 9496.520 ms 2024-04-02T14:56:58.337 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:56:58 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-1[39204]: 2024-04-02T14:56:58.055+0000 7f2315702700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T14:56:58.337 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:56:58 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-1[39204]: 2024-04-02T14:56:58.055+0000 7f2315702700 -1 osd.1 71 *** Got signal Terminated *** 2024-04-02T14:56:58.337 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:56:58 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-1[39204]: 2024-04-02T14:56:58.055+0000 7f2315702700 -1 osd.1 71 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T14:56:59.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 9.72 MB/sec execute 25 sec latency 10496.568 ms 2024-04-02T14:57:00.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 9.34 MB/sec execute 26 sec latency 11496.630 ms 2024-04-02T14:57:01.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 9.00 MB/sec execute 27 sec latency 12496.659 ms 2024-04-02T14:57:02.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 8.67 MB/sec execute 28 sec latency 13496.713 ms 2024-04-02T14:57:03.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 8.38 MB/sec execute 29 sec latency 14496.738 ms 2024-04-02T14:57:03.770 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:57:03 smithi122 podman[57376]: 2024-04-02 14:57:03.508804059 +0000 UTC m=+5.562563119 container died 92d52898245e062f7bae89db3224292c9506cfff8afc069f26a8c83922d2449a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-1, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, distribution-scope=public, vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, io.buildah.version=1.19.8, RELEASE=HEAD, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-02T14:57:04.087 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:57:03 smithi122 podman[57376]: 2024-04-02 14:57:03.870240886 +0000 UTC m=+5.923999935 container cleanup 92d52898245e062f7bae89db3224292c9506cfff8afc069f26a8c83922d2449a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-1, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, name=centos-stream, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, vendor=Red Hat, Inc., release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8) 2024-04-02T14:57:04.087 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:57:03 smithi122 bash[57376]: ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-1 2024-04-02T14:57:04.087 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:57:03 smithi122 bash[57410]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd.1" found: no such container 2024-04-02T14:57:04.294 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 8.10 MB/sec execute 30 sec latency 15496.787 ms 2024-04-02T14:57:04.499 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:57:04 smithi122 podman[57397]: 2024-04-02 14:57:04.121471608 +0000 UTC m=+0.617773316 container remove 92d52898245e062f7bae89db3224292c9506cfff8afc069f26a8c83922d2449a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-1, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, ceph=True, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, release=754, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-02T14:57:04.499 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:57:04 smithi122 podman[57474]: 2024-04-02 14:57:04.228870075 +0000 UTC m=+0.018233051 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T14:57:04.499 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:57:04 smithi122 podman[57474]: 2024-04-02 14:57:04.330647761 +0000 UTC m=+0.120010720 container create e5ee8bb870c20df4c3b35e02725233e3773d7bad24c598b847cfeb18051a066c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-1-deactivate, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, GIT_CLEAN=True, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, RELEASE=HEAD, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, vcs-type=git, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-02T14:57:04.837 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:57:04 smithi122 podman[57474]: 2024-04-02 14:57:04.608481424 +0000 UTC m=+0.397844408 container init e5ee8bb870c20df4c3b35e02725233e3773d7bad24c598b847cfeb18051a066c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-1-deactivate, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, version=8, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, release=754, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream) 2024-04-02T14:57:04.837 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:57:04 smithi122 podman[57474]: 2024-04-02 14:57:04.61933917 +0000 UTC m=+0.408702130 container start e5ee8bb870c20df4c3b35e02725233e3773d7bad24c598b847cfeb18051a066c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-1-deactivate, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.openshift.expose-services=, architecture=x86_64, distribution-scope=public, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, release=754, io.buildah.version=1.19.8, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git) 2024-04-02T14:57:04.837 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:57:04 smithi122 podman[57474]: 2024-04-02 14:57:04.694167071 +0000 UTC m=+0.483530043 container attach e5ee8bb870c20df4c3b35e02725233e3773d7bad24c598b847cfeb18051a066c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-1-deactivate, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.openshift.expose-services=, release=754, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, vcs-type=git, name=centos-stream, RELEASE=HEAD, ceph=True) 2024-04-02T14:57:05.262 INFO:journalctl@ceph.osd.1.smithi122.stdout:Apr 02 14:57:04 smithi122 podman[57474]: 2024-04-02 14:57:04.9779959 +0000 UTC m=+0.767358862 container died e5ee8bb870c20df4c3b35e02725233e3773d7bad24c598b847cfeb18051a066c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-1-deactivate, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, version=8, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, distribution-scope=public, name=centos-stream, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True) 2024-04-02T14:57:05.295 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 7.83 MB/sec execute 31 sec latency 16497.356 ms 2024-04-02T14:57:05.575 DEBUG:teuthology.orchestra.run.smithi122:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.1.service' 2024-04-02T14:57:05.617 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T14:57:05.617 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-02T14:57:05.618 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-02T14:57:05.618 DEBUG:teuthology.orchestra.run.smithi122:> sudo systemctl stop ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.2 2024-04-02T14:57:05.837 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:57:05 smithi122 systemd[1]: Stopping Ceph osd.2 for bc643104-f0ff-11ee-b647-cb9ed24678a4... 2024-04-02T14:57:05.837 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:57:05 smithi122 bash[57552]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd.2" found: no such container 2024-04-02T14:57:06.295 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 7.59 MB/sec execute 32 sec latency 17497.383 ms 2024-04-02T14:57:06.337 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:57:05 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-2[42918]: 2024-04-02T14:57:05.900+0000 7f205a0b1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T14:57:06.337 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:57:05 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-2[42918]: 2024-04-02T14:57:05.900+0000 7f205a0b1700 -1 osd.2 71 *** Got signal Terminated *** 2024-04-02T14:57:06.337 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:57:05 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-2[42918]: 2024-04-02T14:57:05.900+0000 7f205a0b1700 -1 osd.2 71 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T14:57:07.295 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 7.36 MB/sec execute 33 sec latency 18497.415 ms 2024-04-02T14:57:08.295 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 7.14 MB/sec execute 34 sec latency 19497.446 ms 2024-04-02T14:57:09.295 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 6.94 MB/sec execute 35 sec latency 20497.476 ms 2024-04-02T14:57:10.295 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 6.75 MB/sec execute 36 sec latency 21497.520 ms 2024-04-02T14:57:11.295 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 6.56 MB/sec execute 37 sec latency 22497.552 ms 2024-04-02T14:57:11.708 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:57:11 smithi122 podman[57565]: 2024-04-02 14:57:11.440182017 +0000 UTC m=+5.666248825 container died b88321d239b13c652edd3ca41f60f50e594e6e7e1a1fae92446d49e9094a8e45 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-2, version=8, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, architecture=x86_64, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., name=centos-stream) 2024-04-02T14:57:12.059 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:57:11 smithi122 podman[57565]: 2024-04-02 14:57:11.817538606 +0000 UTC m=+6.043605405 container cleanup b88321d239b13c652edd3ca41f60f50e594e6e7e1a1fae92446d49e9094a8e45 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-2, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, release=754, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, ceph=True, vendor=Red Hat, Inc., vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.buildah.version=1.19.8, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public) 2024-04-02T14:57:12.059 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:57:11 smithi122 bash[57565]: ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-2 2024-04-02T14:57:12.059 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:57:11 smithi122 bash[57598]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd.2" found: no such container 2024-04-02T14:57:12.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 6.39 MB/sec execute 38 sec latency 23497.898 ms 2024-04-02T14:57:12.337 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:57:12 smithi122 podman[57586]: 2024-04-02 14:57:12.06919769 +0000 UTC m=+0.625317810 container remove b88321d239b13c652edd3ca41f60f50e594e6e7e1a1fae92446d49e9094a8e45 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-2, release=754, version=8, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, name=centos-stream, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, distribution-scope=public, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True) 2024-04-02T14:57:12.337 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:57:12 smithi122 podman[57660]: 2024-04-02 14:57:12.174513942 +0000 UTC m=+0.019056215 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T14:57:12.337 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:57:12 smithi122 podman[57660]: 2024-04-02 14:57:12.295218239 +0000 UTC m=+0.139760509 container create 3c8634e22c5c6d20476c3df95e680b4d79c0723cf05c710e607adcab4e997a59 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-2-deactivate, maintainer=Guillaume Abrioux , release=754, name=centos-stream, version=8, vcs-type=git, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., RELEASE=HEAD, GIT_CLEAN=True, ceph=True, architecture=x86_64, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0) 2024-04-02T14:57:12.837 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:57:12 smithi122 podman[57660]: 2024-04-02 14:57:12.530988772 +0000 UTC m=+0.375531054 container init 3c8634e22c5c6d20476c3df95e680b4d79c0723cf05c710e607adcab4e997a59 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-2-deactivate, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, vcs-type=git, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., release=754, ceph=True, distribution-scope=public, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True) 2024-04-02T14:57:12.837 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:57:12 smithi122 podman[57660]: 2024-04-02 14:57:12.535542184 +0000 UTC m=+0.380084464 container start 3c8634e22c5c6d20476c3df95e680b4d79c0723cf05c710e607adcab4e997a59 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-2-deactivate, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, version=8, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.openshift.tags=base centos centos-stream, vcs-type=git, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, release=754, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870) 2024-04-02T14:57:12.837 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:57:12 smithi122 podman[57660]: 2024-04-02 14:57:12.614612901 +0000 UTC m=+0.459155183 container attach 3c8634e22c5c6d20476c3df95e680b4d79c0723cf05c710e607adcab4e997a59 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-2-deactivate, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, RELEASE=HEAD, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, vcs-type=git, ceph=True, architecture=x86_64, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , release=754, build-date=2022-05-03T08:36:31.336870) 2024-04-02T14:57:13.182 INFO:journalctl@ceph.osd.2.smithi122.stdout:Apr 02 14:57:12 smithi122 podman[57660]: 2024-04-02 14:57:12.894251621 +0000 UTC m=+0.738793899 container died 3c8634e22c5c6d20476c3df95e680b4d79c0723cf05c710e607adcab4e997a59 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-2-deactivate, release=754, GIT_CLEAN=True, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, distribution-scope=public, vendor=Red Hat, Inc., vcs-type=git, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container) 2024-04-02T14:57:13.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 6.23 MB/sec execute 39 sec latency 24497.941 ms 2024-04-02T14:57:13.506 DEBUG:teuthology.orchestra.run.smithi122:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.2.service' 2024-04-02T14:57:13.562 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T14:57:13.562 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-02T14:57:13.563 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-02T14:57:13.563 DEBUG:teuthology.orchestra.run.smithi122:> sudo systemctl stop ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.3 2024-04-02T14:57:13.717 INFO:journalctl@ceph.osd.3.smithi122.stdout:Apr 02 14:57:13 smithi122 systemd[1]: Stopping Ceph osd.3 for bc643104-f0ff-11ee-b647-cb9ed24678a4... 2024-04-02T14:57:14.087 INFO:journalctl@ceph.osd.3.smithi122.stdout:Apr 02 14:57:13 smithi122 bash[57738]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd.3" found: no such container 2024-04-02T14:57:14.087 INFO:journalctl@ceph.osd.3.smithi122.stdout:Apr 02 14:57:13 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-3[46177]: 2024-04-02T14:57:13.856+0000 7fb724c85700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T14:57:14.087 INFO:journalctl@ceph.osd.3.smithi122.stdout:Apr 02 14:57:13 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-3[46177]: 2024-04-02T14:57:13.856+0000 7fb724c85700 -1 osd.3 71 *** Got signal Terminated *** 2024-04-02T14:57:14.088 INFO:journalctl@ceph.osd.3.smithi122.stdout:Apr 02 14:57:13 smithi122 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-3[46177]: 2024-04-02T14:57:13.856+0000 7fb724c85700 -1 osd.3 71 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T14:57:14.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 6.07 MB/sec execute 40 sec latency 25497.971 ms 2024-04-02T14:57:15.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 5.92 MB/sec execute 41 sec latency 26498.012 ms 2024-04-02T14:57:16.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 5.78 MB/sec execute 42 sec latency 27498.043 ms 2024-04-02T14:57:17.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 5.65 MB/sec execute 43 sec latency 28498.074 ms 2024-04-02T14:57:17.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:17 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:17.550+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:18.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 5.52 MB/sec execute 44 sec latency 29498.109 ms 2024-04-02T14:57:18.398 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:18 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:18.148+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.271739+0000 front 2024-04-02T14:56:53.271763+0000 (oldest deadline 2024-04-02T14:57:17.371976+0000) 2024-04-02T14:57:18.398 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:18 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4[31295]: 2024-04-02T14:57:18.395+0000 7f83734ec700 -1 osd.4 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:54.341179+0000 front 2024-04-02T14:56:54.340988+0000 (oldest deadline 2024-04-02T14:57:17.841120+0000) 2024-04-02T14:57:18.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:18 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:18.562+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:18.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:18 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:18.553+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:19.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 5.40 MB/sec execute 45 sec latency 30498.136 ms 2024-04-02T14:57:19.572 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:19 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4[31295]: 2024-04-02T14:57:19.372+0000 7f83734ec700 -1 osd.4 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:54.341179+0000 front 2024-04-02T14:56:54.340988+0000 (oldest deadline 2024-04-02T14:57:17.841120+0000) 2024-04-02T14:57:19.573 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:19 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:19.179+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.271739+0000 front 2024-04-02T14:56:53.271763+0000 (oldest deadline 2024-04-02T14:57:17.371976+0000) 2024-04-02T14:57:19.762 INFO:journalctl@ceph.osd.3.smithi122.stdout:Apr 02 14:57:19 smithi122 podman[57749]: 2024-04-02 14:57:19.479810099 +0000 UTC m=+5.743917176 container died 1aec61f33b52d534044d4beb463fd27ac162bc672c8a30563b865a1ad94a03c5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-3, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , GIT_CLEAN=True, architecture=x86_64, release=754, io.openshift.tags=base centos centos-stream, name=centos-stream, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, RELEASE=HEAD, version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-02T14:57:19.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:19 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:19.582+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:19.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:19 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:19.592+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:20.087 INFO:journalctl@ceph.osd.3.smithi122.stdout:Apr 02 14:57:19 smithi122 podman[57749]: 2024-04-02 14:57:19.862804343 +0000 UTC m=+6.126911418 container cleanup 1aec61f33b52d534044d4beb463fd27ac162bc672c8a30563b865a1ad94a03c5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-3, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.openshift.expose-services=, GIT_CLEAN=True, release=754, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public) 2024-04-02T14:57:20.087 INFO:journalctl@ceph.osd.3.smithi122.stdout:Apr 02 14:57:19 smithi122 bash[57749]: ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-3 2024-04-02T14:57:20.087 INFO:journalctl@ceph.osd.3.smithi122.stdout:Apr 02 14:57:19 smithi122 bash[57778]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd.3" found: no such container 2024-04-02T14:57:20.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 5.28 MB/sec execute 46 sec latency 31498.166 ms 2024-04-02T14:57:20.455 INFO:journalctl@ceph.osd.3.smithi122.stdout:Apr 02 14:57:20 smithi122 podman[57766]: 2024-04-02 14:57:20.122627353 +0000 UTC m=+0.632724639 container remove 1aec61f33b52d534044d4beb463fd27ac162bc672c8a30563b865a1ad94a03c5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-3, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, release=754, maintainer=Guillaume Abrioux , ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, io.openshift.tags=base centos centos-stream, architecture=x86_64, io.buildah.version=1.19.8, version=8, name=centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-02T14:57:20.455 INFO:journalctl@ceph.osd.3.smithi122.stdout:Apr 02 14:57:20 smithi122 podman[57841]: 2024-04-02 14:57:20.227120438 +0000 UTC m=+0.018426260 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T14:57:20.456 INFO:journalctl@ceph.osd.3.smithi122.stdout:Apr 02 14:57:20 smithi122 podman[57841]: 2024-04-02 14:57:20.323421887 +0000 UTC m=+0.114727705 container create d24adeeb4a603457c53dd405b4569857f7540ea5f4ed537481f3d6b63fc0dd57 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-3-deactivate, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_CLEAN=True, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, release=754, version=8, com.redhat.component=centos-stream-container, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, name=centos-stream) 2024-04-02T14:57:20.556 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:20 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4[31295]: 2024-04-02T14:57:20.331+0000 7f83734ec700 -1 osd.4 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:54.341179+0000 front 2024-04-02T14:56:54.340988+0000 (oldest deadline 2024-04-02T14:57:17.841120+0000) 2024-04-02T14:57:20.556 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:20 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:20.203+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.271739+0000 front 2024-04-02T14:56:53.271763+0000 (oldest deadline 2024-04-02T14:57:17.371976+0000) 2024-04-02T14:57:20.837 INFO:journalctl@ceph.osd.3.smithi122.stdout:Apr 02 14:57:20 smithi122 podman[57841]: 2024-04-02 14:57:20.551493835 +0000 UTC m=+0.342799657 container init d24adeeb4a603457c53dd405b4569857f7540ea5f4ed537481f3d6b63fc0dd57 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-3-deactivate, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, architecture=x86_64, GIT_BRANCH=HEAD, io.openshift.expose-services=, vendor=Red Hat, Inc., version=8, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-02T14:57:20.838 INFO:journalctl@ceph.osd.3.smithi122.stdout:Apr 02 14:57:20 smithi122 podman[57841]: 2024-04-02 14:57:20.560988331 +0000 UTC m=+0.352294146 container start d24adeeb4a603457c53dd405b4569857f7540ea5f4ed537481f3d6b63fc0dd57 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-3-deactivate, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, version=8, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.buildah.version=1.19.8, RELEASE=HEAD, name=centos-stream, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, build-date=2022-05-03T08:36:31.336870, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-02T14:57:20.838 INFO:journalctl@ceph.osd.3.smithi122.stdout:Apr 02 14:57:20 smithi122 podman[57841]: 2024-04-02 14:57:20.649215954 +0000 UTC m=+0.440521774 container attach d24adeeb4a603457c53dd405b4569857f7540ea5f4ed537481f3d6b63fc0dd57 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-3-deactivate, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, com.redhat.component=centos-stream-container, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, name=centos-stream) 2024-04-02T14:57:20.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:20 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:20.565+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:20.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:20 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:20.642+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:21.225 INFO:journalctl@ceph.osd.3.smithi122.stdout:Apr 02 14:57:20 smithi122 podman[57841]: 2024-04-02 14:57:20.919404402 +0000 UTC m=+0.710710222 container died d24adeeb4a603457c53dd405b4569857f7540ea5f4ed537481f3d6b63fc0dd57 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-3-deactivate, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, release=754, name=centos-stream, architecture=x86_64, RELEASE=HEAD, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.buildah.version=1.19.8) 2024-04-02T14:57:21.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 5.17 MB/sec execute 47 sec latency 32498.200 ms 2024-04-02T14:57:21.547 DEBUG:teuthology.orchestra.run.smithi122:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.3.service' 2024-04-02T14:57:21.586 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T14:57:21.586 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-02T14:57:21.586 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-02T14:57:21.587 DEBUG:teuthology.orchestra.run.smithi183:> sudo systemctl stop ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.4 2024-04-02T14:57:21.603 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:21 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4[31295]: 2024-04-02T14:57:21.352+0000 7f83734ec700 -1 osd.4 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:54.341179+0000 front 2024-04-02T14:56:54.340988+0000 (oldest deadline 2024-04-02T14:57:17.841120+0000) 2024-04-02T14:57:21.603 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:21 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:21.220+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.271739+0000 front 2024-04-02T14:56:53.271763+0000 (oldest deadline 2024-04-02T14:57:17.371976+0000) 2024-04-02T14:57:21.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:21 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:21.613+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:21.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:21 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:21.614+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:21.899 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:21 smithi183 systemd[1]: Stopping Ceph osd.4 for bc643104-f0ff-11ee-b647-cb9ed24678a4... 2024-04-02T14:57:21.899 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:21 smithi183 bash[44437]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd.4" found: no such container 2024-04-02T14:57:21.899 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:21 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4[31295]: 2024-04-02T14:57:21.816+0000 7f8377352700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T14:57:21.899 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:21 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4[31295]: 2024-04-02T14:57:21.816+0000 7f8377352700 -1 osd.4 71 *** Got signal Terminated *** 2024-04-02T14:57:21.899 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:21 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4[31295]: 2024-04-02T14:57:21.816+0000 7f8377352700 -1 osd.4 71 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T14:57:22.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 5.06 MB/sec execute 48 sec latency 33498.229 ms 2024-04-02T14:57:22.556 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:22 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4[31295]: 2024-04-02T14:57:22.318+0000 7f83734ec700 -1 osd.4 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:54.341179+0000 front 2024-04-02T14:56:54.340988+0000 (oldest deadline 2024-04-02T14:57:17.841120+0000) 2024-04-02T14:57:22.556 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:22 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:22.200+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.271739+0000 front 2024-04-02T14:56:53.271763+0000 (oldest deadline 2024-04-02T14:57:17.371976+0000) 2024-04-02T14:57:22.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:22 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:22.565+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:22.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:22 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:22.602+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:23.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 4.96 MB/sec execute 49 sec latency 34498.263 ms 2024-04-02T14:57:23.590 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:23 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4[31295]: 2024-04-02T14:57:23.275+0000 7f83734ec700 -1 osd.4 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:54.341179+0000 front 2024-04-02T14:56:54.340988+0000 (oldest deadline 2024-04-02T14:57:17.841120+0000) 2024-04-02T14:57:23.591 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:23 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:23.208+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.271739+0000 front 2024-04-02T14:56:53.271763+0000 (oldest deadline 2024-04-02T14:57:17.371976+0000) 2024-04-02T14:57:23.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:23 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:23.600+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:23.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:23 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:23.646+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:23.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:23 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:23.646+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:24.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 4.86 MB/sec execute 50 sec latency 35498.294 ms 2024-04-02T14:57:24.634 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:24 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4[31295]: 2024-04-02T14:57:24.262+0000 7f83734ec700 -1 osd.4 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:54.341179+0000 front 2024-04-02T14:56:54.340988+0000 (oldest deadline 2024-04-02T14:57:17.841120+0000) 2024-04-02T14:57:24.634 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:24 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:24.252+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.271739+0000 front 2024-04-02T14:56:53.271763+0000 (oldest deadline 2024-04-02T14:57:17.371976+0000) 2024-04-02T14:57:24.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:24 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:24.647+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:24.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:24 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:24.643+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:24.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:24 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:24.643+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:25.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 4.76 MB/sec execute 51 sec latency 36498.336 ms 2024-04-02T14:57:25.621 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:25 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4[31295]: 2024-04-02T14:57:25.255+0000 7f83734ec700 -1 osd.4 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:54.341179+0000 front 2024-04-02T14:56:54.340988+0000 (oldest deadline 2024-04-02T14:57:17.841120+0000) 2024-04-02T14:57:25.622 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:25 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:25.232+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.271739+0000 front 2024-04-02T14:56:53.271763+0000 (oldest deadline 2024-04-02T14:57:17.371976+0000) 2024-04-02T14:57:25.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:25 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:25.631+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:25.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:25 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:25.631+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:25.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:25 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:25.641+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:26.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 4.67 MB/sec execute 52 sec latency 37498.365 ms 2024-04-02T14:57:26.589 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:26 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4[31295]: 2024-04-02T14:57:26.299+0000 7f83734ec700 -1 osd.4 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:54.341179+0000 front 2024-04-02T14:56:54.340988+0000 (oldest deadline 2024-04-02T14:57:17.841120+0000) 2024-04-02T14:57:26.589 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:26 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:26.205+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.271739+0000 front 2024-04-02T14:56:53.271763+0000 (oldest deadline 2024-04-02T14:57:17.371976+0000) 2024-04-02T14:57:26.590 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:26 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:26.205+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:59.673146+0000 front 2024-04-02T14:56:59.673214+0000 (oldest deadline 2024-04-02T14:57:25.473719+0000) 2024-04-02T14:57:26.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:26 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:26.656+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:26.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:26 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:26.599+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:26.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:26 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:26.599+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:27.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 4.58 MB/sec execute 53 sec latency 38498.392 ms 2024-04-02T14:57:27.613 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:27 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:27.162+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.271739+0000 front 2024-04-02T14:56:53.271763+0000 (oldest deadline 2024-04-02T14:57:17.371976+0000) 2024-04-02T14:57:27.613 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:27 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:27.162+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:59.673146+0000 front 2024-04-02T14:56:59.673214+0000 (oldest deadline 2024-04-02T14:57:25.473719+0000) 2024-04-02T14:57:27.614 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:27 smithi183 podman[44448]: 2024-04-02 14:57:27.358920698 +0000 UTC m=+5.672096887 container died 0d1dbe82a34282a317d7daaff05265169d31747886dc15ca9457dee8b019b6cb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, name=centos-stream, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , distribution-scope=public, version=8, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-02T14:57:27.898 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:27 smithi183 podman[44448]: 2024-04-02 14:57:27.849779413 +0000 UTC m=+6.162955642 container cleanup 0d1dbe82a34282a317d7daaff05265169d31747886dc15ca9457dee8b019b6cb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4, io.openshift.tags=base centos centos-stream, version=8, RELEASE=HEAD, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, GIT_CLEAN=True, architecture=x86_64, name=centos-stream, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True) 2024-04-02T14:57:27.898 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:27 smithi183 bash[44448]: ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4 2024-04-02T14:57:27.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:27 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:27.622+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:27.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:27 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:27.622+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:27.899 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:27 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:27.636+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:27.899 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:27 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:27.636+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:57:01.106781+0000 front 2024-04-02T14:57:01.106741+0000 (oldest deadline 2024-04-02T14:57:27.006684+0000) 2024-04-02T14:57:28.273 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:28 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:28.196+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.271739+0000 front 2024-04-02T14:56:53.271763+0000 (oldest deadline 2024-04-02T14:57:17.371976+0000) 2024-04-02T14:57:28.273 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:28 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:28.196+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:59.673146+0000 front 2024-04-02T14:56:59.673214+0000 (oldest deadline 2024-04-02T14:57:25.473719+0000) 2024-04-02T14:57:28.274 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:28 smithi183 podman[44466]: 2024-04-02 14:57:28.026107537 +0000 UTC m=+0.663619567 container remove 0d1dbe82a34282a317d7daaff05265169d31747886dc15ca9457dee8b019b6cb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4, GIT_CLEAN=True, architecture=x86_64, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., io.openshift.expose-services=, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-02T14:57:28.274 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:28 smithi183 bash[44480]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd.4" found: no such container 2024-04-02T14:57:28.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 4.50 MB/sec execute 54 sec latency 39498.420 ms 2024-04-02T14:57:28.588 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:28 smithi183 podman[44539]: 2024-04-02 14:57:28.187410048 +0000 UTC m=+0.018561266 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T14:57:28.588 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:28 smithi183 podman[44539]: 2024-04-02 14:57:28.285327677 +0000 UTC m=+0.116478894 container create 049d02b7cc933973d56db371ce8cc770246f9be71bfa80c0440a20aa9b0a1c7e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4-deactivate, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, name=centos-stream, io.openshift.expose-services=, GIT_CLEAN=True, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, vcs-type=git) 2024-04-02T14:57:28.588 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:28 smithi183 podman[44539]: 2024-04-02 14:57:28.52957157 +0000 UTC m=+0.360722788 container init 049d02b7cc933973d56db371ce8cc770246f9be71bfa80c0440a20aa9b0a1c7e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4-deactivate, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, io.buildah.version=1.19.8, vcs-type=git, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , release=754, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-02T14:57:28.589 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:28 smithi183 podman[44539]: 2024-04-02 14:57:28.532641636 +0000 UTC m=+0.363792857 container start 049d02b7cc933973d56db371ce8cc770246f9be71bfa80c0440a20aa9b0a1c7e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4-deactivate, architecture=x86_64, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, ceph=True, maintainer=Guillaume Abrioux , io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, distribution-scope=public, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, version=8) 2024-04-02T14:57:28.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:28 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:28.671+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:28.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:28 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:28.671+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:57:01.106781+0000 front 2024-04-02T14:57:01.106741+0000 (oldest deadline 2024-04-02T14:57:27.006684+0000) 2024-04-02T14:57:28.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:28 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:28.597+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:28.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:28 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:28.597+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:28.899 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:28 smithi183 podman[44539]: 2024-04-02 14:57:28.604408851 +0000 UTC m=+0.435560086 container attach 049d02b7cc933973d56db371ce8cc770246f9be71bfa80c0440a20aa9b0a1c7e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4-deactivate, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, vcs-type=git, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, ceph=True, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, GIT_CLEAN=True) 2024-04-02T14:57:28.899 INFO:journalctl@ceph.osd.4.smithi183.stdout:Apr 02 14:57:28 smithi183 podman[44539]: 2024-04-02 14:57:28.821439459 +0000 UTC m=+0.652590677 container died 049d02b7cc933973d56db371ce8cc770246f9be71bfa80c0440a20aa9b0a1c7e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-4-deactivate, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, ceph=True, RELEASE=HEAD, version=8, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, name=centos-stream, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-02T14:57:29.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 4.42 MB/sec execute 55 sec latency 40498.450 ms 2024-04-02T14:57:29.398 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:29 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:29.170+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.271739+0000 front 2024-04-02T14:56:53.271763+0000 (oldest deadline 2024-04-02T14:57:17.371976+0000) 2024-04-02T14:57:29.405 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:29 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:29.170+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:59.673146+0000 front 2024-04-02T14:56:59.673214+0000 (oldest deadline 2024-04-02T14:57:25.473719+0000) 2024-04-02T14:57:29.424 DEBUG:teuthology.orchestra.run.smithi183:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.4.service' 2024-04-02T14:57:29.506 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T14:57:29.506 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-02T14:57:29.506 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-02T14:57:29.507 DEBUG:teuthology.orchestra.run.smithi183:> sudo systemctl stop ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.5 2024-04-02T14:57:29.648 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:29 smithi183 systemd[1]: Stopping Ceph osd.5 for bc643104-f0ff-11ee-b647-cb9ed24678a4... 2024-04-02T14:57:29.648 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:29 smithi183 bash[44620]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd.5" found: no such container 2024-04-02T14:57:29.648 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:29 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:29.573+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:29.648 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:29 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:29.573+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:30.148 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:29 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:29.697+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:30.148 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:29 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:29.697+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:57:01.106781+0000 front 2024-04-02T14:57:01.106741+0000 (oldest deadline 2024-04-02T14:57:27.006684+0000) 2024-04-02T14:57:30.148 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:29 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:29.769+0000 7f892bcf9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T14:57:30.148 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:29 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:29.769+0000 7f892bcf9700 -1 osd.5 71 *** Got signal Terminated *** 2024-04-02T14:57:30.149 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:29 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:29.769+0000 7f892bcf9700 -1 osd.5 71 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T14:57:30.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 4.34 MB/sec execute 56 sec latency 41498.477 ms 2024-04-02T14:57:30.566 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:30 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:30.172+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.271739+0000 front 2024-04-02T14:56:53.271763+0000 (oldest deadline 2024-04-02T14:57:17.371976+0000) 2024-04-02T14:57:30.566 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:30 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:30.172+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:59.673146+0000 front 2024-04-02T14:56:59.673214+0000 (oldest deadline 2024-04-02T14:57:25.473719+0000) 2024-04-02T14:57:30.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:30 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:30.726+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:30.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:30 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:30.726+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:57:01.106781+0000 front 2024-04-02T14:57:01.106741+0000 (oldest deadline 2024-04-02T14:57:27.006684+0000) 2024-04-02T14:57:30.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:30 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:30.575+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:30.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:30 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:30.575+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:31.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 4.26 MB/sec execute 57 sec latency 42498.509 ms 2024-04-02T14:57:31.398 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:31 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:31.157+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.271739+0000 front 2024-04-02T14:56:53.271763+0000 (oldest deadline 2024-04-02T14:57:17.371976+0000) 2024-04-02T14:57:31.398 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:31 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:31.157+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:59.673146+0000 front 2024-04-02T14:56:59.673214+0000 (oldest deadline 2024-04-02T14:57:25.473719+0000) 2024-04-02T14:57:31.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:31 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:31.697+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:31.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:31 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:31.697+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:57:01.106781+0000 front 2024-04-02T14:57:01.106741+0000 (oldest deadline 2024-04-02T14:57:27.006684+0000) 2024-04-02T14:57:31.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:31 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:31.624+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:31.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:31 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:31.624+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:32.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 4.19 MB/sec execute 58 sec latency 43498.553 ms 2024-04-02T14:57:32.629 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:32 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:32.206+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.271739+0000 front 2024-04-02T14:56:53.271763+0000 (oldest deadline 2024-04-02T14:57:17.371976+0000) 2024-04-02T14:57:32.629 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:32 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:32.206+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:59.673146+0000 front 2024-04-02T14:56:59.673214+0000 (oldest deadline 2024-04-02T14:57:25.473719+0000) 2024-04-02T14:57:32.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:32 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:32.708+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:32.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:32 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:32.708+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:57:01.106781+0000 front 2024-04-02T14:57:01.106741+0000 (oldest deadline 2024-04-02T14:57:27.006684+0000) 2024-04-02T14:57:32.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:32 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:32.639+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:32.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:32 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:32.639+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:33.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 4.12 MB/sec execute 59 sec latency 44498.582 ms 2024-04-02T14:57:33.600 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:33 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:33.172+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.271739+0000 front 2024-04-02T14:56:53.271763+0000 (oldest deadline 2024-04-02T14:57:17.371976+0000) 2024-04-02T14:57:33.600 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:33 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:33.172+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:59.673146+0000 front 2024-04-02T14:56:59.673214+0000 (oldest deadline 2024-04-02T14:57:25.473719+0000) 2024-04-02T14:57:33.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:33 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:33.756+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:33.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:33 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:33.756+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:57:01.106781+0000 front 2024-04-02T14:57:01.106741+0000 (oldest deadline 2024-04-02T14:57:27.006684+0000) 2024-04-02T14:57:33.902 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:33 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:33.609+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:33.902 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:33 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:33.609+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:33.903 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:33 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:33.609+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:08.302224+0000 front 2024-04-02T14:57:08.302114+0000 (oldest deadline 2024-04-02T14:57:33.602223+0000) 2024-04-02T14:57:34.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 4.05 MB/sec execute 60 sec latency 45498.610 ms 2024-04-02T14:57:34.603 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:34 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:34.189+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.271739+0000 front 2024-04-02T14:56:53.271763+0000 (oldest deadline 2024-04-02T14:57:17.371976+0000) 2024-04-02T14:57:34.603 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:34 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:34.189+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:59.673146+0000 front 2024-04-02T14:56:59.673214+0000 (oldest deadline 2024-04-02T14:57:25.473719+0000) 2024-04-02T14:57:34.603 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:34 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5[34270]: 2024-04-02T14:57:34.189+0000 7f8926690700 -1 osd.5 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:10.174870+0000 front 2024-04-02T14:57:10.174832+0000 (oldest deadline 2024-04-02T14:57:33.575547+0000) 2024-04-02T14:57:34.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:34 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:34.719+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:34.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:34 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:34.719+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:57:01.106781+0000 front 2024-04-02T14:57:01.106741+0000 (oldest deadline 2024-04-02T14:57:27.006684+0000) 2024-04-02T14:57:34.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:34 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:34.612+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:34.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:34 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:34.612+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:34.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:34 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:34.612+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:08.302224+0000 front 2024-04-02T14:57:08.302114+0000 (oldest deadline 2024-04-02T14:57:33.602223+0000) 2024-04-02T14:57:35.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.98 MB/sec execute 61 sec latency 46498.655 ms 2024-04-02T14:57:35.512 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:35 smithi183 podman[44633]: 2024-04-02 14:57:35.246312771 +0000 UTC m=+5.610627234 container died 9bc6c62fa5cb568c60d510593a0847df16722b35cbe463984eac6fdff2613156 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., ceph=True, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, version=8, GIT_BRANCH=HEAD, RELEASE=HEAD, distribution-scope=public, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git) 2024-04-02T14:57:35.898 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:35 smithi183 podman[44633]: 2024-04-02 14:57:35.713256182 +0000 UTC m=+6.077570633 container cleanup 9bc6c62fa5cb568c60d510593a0847df16722b35cbe463984eac6fdff2613156 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , version=8, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8) 2024-04-02T14:57:35.898 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:35 smithi183 bash[44633]: ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5 2024-04-02T14:57:35.898 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:35 smithi183 bash[44673]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd.5" found: no such container 2024-04-02T14:57:35.899 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:35 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:35.731+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:35.899 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:35 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:35.731+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:57:01.106781+0000 front 2024-04-02T14:57:01.106741+0000 (oldest deadline 2024-04-02T14:57:27.006684+0000) 2024-04-02T14:57:35.899 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:35 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:35.731+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:09.307610+0000 front 2024-04-02T14:57:09.307642+0000 (oldest deadline 2024-04-02T14:57:35.207671+0000) 2024-04-02T14:57:35.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:35 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:35.583+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:35.900 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:35 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:35.583+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:35.900 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:35 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:35.583+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:08.302224+0000 front 2024-04-02T14:57:08.302114+0000 (oldest deadline 2024-04-02T14:57:33.602223+0000) 2024-04-02T14:57:36.280 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:35 smithi183 podman[44661]: 2024-04-02 14:57:35.989747367 +0000 UTC m=+0.743446945 container remove 9bc6c62fa5cb568c60d510593a0847df16722b35cbe463984eac6fdff2613156 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.openshift.tags=base centos centos-stream, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc.) 2024-04-02T14:57:36.280 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:36 smithi183 podman[44741]: 2024-04-02 14:57:36.085473556 +0000 UTC m=+0.018519785 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T14:57:36.280 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:36 smithi183 podman[44741]: 2024-04-02 14:57:36.19073004 +0000 UTC m=+0.123776286 container create 602f5a0974c60740589d725e6fc041f0bbeced0f0f7836a0baad3bc585e61adb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5-deactivate, io.openshift.expose-services=, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., distribution-scope=public, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, version=8, vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream) 2024-04-02T14:57:36.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.92 MB/sec execute 62 sec latency 47498.679 ms 2024-04-02T14:57:36.612 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:36 smithi183 podman[44741]: 2024-04-02 14:57:36.409669635 +0000 UTC m=+0.342715837 container init 602f5a0974c60740589d725e6fc041f0bbeced0f0f7836a0baad3bc585e61adb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5-deactivate, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, com.redhat.component=centos-stream-container, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.openshift.expose-services=, architecture=x86_64, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-02T14:57:36.612 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:36 smithi183 podman[44741]: 2024-04-02 14:57:36.412687703 +0000 UTC m=+0.345733904 container start 602f5a0974c60740589d725e6fc041f0bbeced0f0f7836a0baad3bc585e61adb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5-deactivate, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, vcs-type=git, vendor=Red Hat, Inc., distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , release=754, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-02T14:57:36.612 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:36 smithi183 podman[44741]: 2024-04-02 14:57:36.467739243 +0000 UTC m=+0.400785457 container attach 602f5a0974c60740589d725e6fc041f0bbeced0f0f7836a0baad3bc585e61adb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5-deactivate, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, io.k8s.display-name=CentOS Stream 8, ceph=True, distribution-scope=public, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, io.openshift.expose-services=, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux ) 2024-04-02T14:57:36.898 INFO:journalctl@ceph.osd.5.smithi183.stdout:Apr 02 14:57:36 smithi183 podman[44741]: 2024-04-02 14:57:36.702016628 +0000 UTC m=+0.635062843 container died 602f5a0974c60740589d725e6fc041f0bbeced0f0f7836a0baad3bc585e61adb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-5-deactivate, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, vendor=Red Hat, Inc., release=754, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.openshift.expose-services=, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True) 2024-04-02T14:57:36.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:36 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:36.709+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:36.899 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:36 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:36.709+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:57:01.106781+0000 front 2024-04-02T14:57:01.106741+0000 (oldest deadline 2024-04-02T14:57:27.006684+0000) 2024-04-02T14:57:36.899 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:36 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:36.709+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:09.307610+0000 front 2024-04-02T14:57:09.307642+0000 (oldest deadline 2024-04-02T14:57:35.207671+0000) 2024-04-02T14:57:36.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:36 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:36.621+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:36.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:36 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:36.621+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:36.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:36 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:36.621+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:08.302224+0000 front 2024-04-02T14:57:08.302114+0000 (oldest deadline 2024-04-02T14:57:33.602223+0000) 2024-04-02T14:57:37.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.86 MB/sec execute 63 sec latency 48498.720 ms 2024-04-02T14:57:37.321 DEBUG:teuthology.orchestra.run.smithi183:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.5.service' 2024-04-02T14:57:37.361 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T14:57:37.361 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-02T14:57:37.362 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-02T14:57:37.362 DEBUG:teuthology.orchestra.run.smithi183:> sudo systemctl stop ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.6 2024-04-02T14:57:37.580 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:37 smithi183 systemd[1]: Stopping Ceph osd.6 for bc643104-f0ff-11ee-b647-cb9ed24678a4... 2024-04-02T14:57:37.580 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:37 smithi183 bash[44820]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd.6" found: no such container 2024-04-02T14:57:37.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:37 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:37.609+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:37.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:37 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:37.609+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:37.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:37 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:37.609+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:08.302224+0000 front 2024-04-02T14:57:08.302114+0000 (oldest deadline 2024-04-02T14:57:33.602223+0000) 2024-04-02T14:57:37.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:37 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:37.589+0000 7ff68068c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T14:57:37.899 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:37 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:37.589+0000 7ff68068c700 -1 osd.6 71 *** Got signal Terminated *** 2024-04-02T14:57:37.899 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:37 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:37.589+0000 7ff68068c700 -1 osd.6 71 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T14:57:37.899 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:37 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:37.714+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:37.899 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:37 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:37.714+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:57:01.106781+0000 front 2024-04-02T14:57:01.106741+0000 (oldest deadline 2024-04-02T14:57:27.006684+0000) 2024-04-02T14:57:37.899 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:37 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:37.714+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:09.307610+0000 front 2024-04-02T14:57:09.307642+0000 (oldest deadline 2024-04-02T14:57:35.207671+0000) 2024-04-02T14:57:38.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.80 MB/sec execute 64 sec latency 49498.748 ms 2024-04-02T14:57:38.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:38 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:38.738+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:38.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:38 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:38.738+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:57:01.106781+0000 front 2024-04-02T14:57:01.106741+0000 (oldest deadline 2024-04-02T14:57:27.006684+0000) 2024-04-02T14:57:38.899 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:38 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:38.738+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:09.307610+0000 front 2024-04-02T14:57:09.307642+0000 (oldest deadline 2024-04-02T14:57:35.207671+0000) 2024-04-02T14:57:38.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:38 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:38.585+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:38.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:38 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:38.585+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:38.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:38 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:38.585+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:08.302224+0000 front 2024-04-02T14:57:08.302114+0000 (oldest deadline 2024-04-02T14:57:33.602223+0000) 2024-04-02T14:57:39.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.74 MB/sec execute 65 sec latency 50498.772 ms 2024-04-02T14:57:39.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:39 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:39.722+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:39.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:39 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:39.722+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:57:01.106781+0000 front 2024-04-02T14:57:01.106741+0000 (oldest deadline 2024-04-02T14:57:27.006684+0000) 2024-04-02T14:57:39.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:39 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:39.722+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:09.307610+0000 front 2024-04-02T14:57:09.307642+0000 (oldest deadline 2024-04-02T14:57:35.207671+0000) 2024-04-02T14:57:39.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:39 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:39.629+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:39.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:39 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:39.629+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:39.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:39 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:39.629+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:08.302224+0000 front 2024-04-02T14:57:08.302114+0000 (oldest deadline 2024-04-02T14:57:33.602223+0000) 2024-04-02T14:57:39.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:39 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:39.629+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6830 osd.3 since back 2024-04-02T14:57:13.602763+0000 front 2024-04-02T14:57:13.605907+0000 (oldest deadline 2024-04-02T14:57:39.502866+0000) 2024-04-02T14:57:40.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.68 MB/sec execute 66 sec latency 51498.811 ms 2024-04-02T14:57:40.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:40 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:40.746+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:40.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:40 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:40.746+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:57:01.106781+0000 front 2024-04-02T14:57:01.106741+0000 (oldest deadline 2024-04-02T14:57:27.006684+0000) 2024-04-02T14:57:40.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:40 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:40.746+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:09.307610+0000 front 2024-04-02T14:57:09.307642+0000 (oldest deadline 2024-04-02T14:57:35.207671+0000) 2024-04-02T14:57:40.899 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:40 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:40.746+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6830 osd.3 since back 2024-04-02T14:57:15.208176+0000 front 2024-04-02T14:57:15.208163+0000 (oldest deadline 2024-04-02T14:57:40.508132+0000) 2024-04-02T14:57:40.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:40 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:40.583+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:40.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:40 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:40.583+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:40.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:40 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:40.583+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:08.302224+0000 front 2024-04-02T14:57:08.302114+0000 (oldest deadline 2024-04-02T14:57:33.602223+0000) 2024-04-02T14:57:40.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:40 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:40.583+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6830 osd.3 since back 2024-04-02T14:57:13.602763+0000 front 2024-04-02T14:57:13.605907+0000 (oldest deadline 2024-04-02T14:57:39.502866+0000) 2024-04-02T14:57:41.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.63 MB/sec execute 67 sec latency 52498.840 ms 2024-04-02T14:57:41.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:41 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:41.774+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:53.005276+0000 front 2024-04-02T14:56:53.005262+0000 (oldest deadline 2024-04-02T14:57:17.705357+0000) 2024-04-02T14:57:41.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:41 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:41.774+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:57:01.106781+0000 front 2024-04-02T14:57:01.106741+0000 (oldest deadline 2024-04-02T14:57:27.006684+0000) 2024-04-02T14:57:41.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:41 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:41.774+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:09.307610+0000 front 2024-04-02T14:57:09.307642+0000 (oldest deadline 2024-04-02T14:57:35.207671+0000) 2024-04-02T14:57:41.898 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:41 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6[37325]: 2024-04-02T14:57:41.774+0000 7ff67d828700 -1 osd.6 71 heartbeat_check: no reply from 172.21.15.122:6830 osd.3 since back 2024-04-02T14:57:15.208176+0000 front 2024-04-02T14:57:15.208163+0000 (oldest deadline 2024-04-02T14:57:40.508132+0000) 2024-04-02T14:57:41.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:41 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:41.585+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:41.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:41 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:41.585+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:41.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:41 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:41.585+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:08.302224+0000 front 2024-04-02T14:57:08.302114+0000 (oldest deadline 2024-04-02T14:57:33.602223+0000) 2024-04-02T14:57:41.899 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:41 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:41.585+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6830 osd.3 since back 2024-04-02T14:57:13.602763+0000 front 2024-04-02T14:57:13.605907+0000 (oldest deadline 2024-04-02T14:57:39.502866+0000) 2024-04-02T14:57:42.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.57 MB/sec execute 68 sec latency 53498.864 ms 2024-04-02T14:57:42.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:42 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:42.540+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:42.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:42 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:42.540+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:42.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:42 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:42.540+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:08.302224+0000 front 2024-04-02T14:57:08.302114+0000 (oldest deadline 2024-04-02T14:57:33.602223+0000) 2024-04-02T14:57:42.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:42 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:42.540+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6830 osd.3 since back 2024-04-02T14:57:13.602763+0000 front 2024-04-02T14:57:13.605907+0000 (oldest deadline 2024-04-02T14:57:39.502866+0000) 2024-04-02T14:57:43.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.52 MB/sec execute 69 sec latency 54498.922 ms 2024-04-02T14:57:43.397 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:43 smithi183 podman[44830]: 2024-04-02 14:57:43.153165166 +0000 UTC m=+5.669705971 container died f9c8a0a7bfdc37325ce7935abd04a71c0fcce719d72b6cf616adeffd27434cc6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, release=754, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, com.redhat.component=centos-stream-container, name=centos-stream, vcs-type=git) 2024-04-02T14:57:43.648 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:43 smithi183 podman[44830]: 2024-04-02 14:57:43.59859951 +0000 UTC m=+6.115140313 container cleanup f9c8a0a7bfdc37325ce7935abd04a71c0fcce719d72b6cf616adeffd27434cc6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , RELEASE=HEAD, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., vcs-type=git, distribution-scope=public, version=8, architecture=x86_64, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_CLEAN=True, ceph=True, build-date=2022-05-03T08:36:31.336870) 2024-04-02T14:57:43.648 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:43 smithi183 bash[44830]: ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6 2024-04-02T14:57:43.648 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:43 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:43.529+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:43.649 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:43 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:43.529+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:43.649 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:43 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:43.529+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:08.302224+0000 front 2024-04-02T14:57:08.302114+0000 (oldest deadline 2024-04-02T14:57:33.602223+0000) 2024-04-02T14:57:43.649 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:43 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:43.529+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6830 osd.3 since back 2024-04-02T14:57:13.602763+0000 front 2024-04-02T14:57:13.605907+0000 (oldest deadline 2024-04-02T14:57:39.502866+0000) 2024-04-02T14:57:44.048 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:43 smithi183 podman[44849]: 2024-04-02 14:57:43.80008408 +0000 UTC m=+0.644772526 container remove f9c8a0a7bfdc37325ce7935abd04a71c0fcce719d72b6cf616adeffd27434cc6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, distribution-scope=public, architecture=x86_64, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, release=754, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, ceph=True, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-02T14:57:44.048 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:43 smithi183 bash[44862]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd.6" found: no such container 2024-04-02T14:57:44.049 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:44 smithi183 podman[44927]: 2024-04-02 14:57:43.943149234 +0000 UTC m=+0.019012088 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T14:57:44.049 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:44 smithi183 podman[44927]: 2024-04-02 14:57:44.059045452 +0000 UTC m=+0.134908288 container create d2d48430df7c70c606680d848e4aa5e1c1402c615f99671d9d476ba56a4dcf42 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6-deactivate, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, version=8, GIT_BRANCH=HEAD, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, release=754, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-02T14:57:44.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.47 MB/sec execute 70 sec latency 55498.954 ms 2024-04-02T14:57:44.342 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:44 smithi183 podman[44927]: 2024-04-02 14:57:44.27860212 +0000 UTC m=+0.354464973 container init d2d48430df7c70c606680d848e4aa5e1c1402c615f99671d9d476ba56a4dcf42 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6-deactivate, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, vcs-type=git, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, distribution-scope=public, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., architecture=x86_64, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.buildah.version=1.19.8, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-02T14:57:44.342 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:44 smithi183 podman[44927]: 2024-04-02 14:57:44.281877356 +0000 UTC m=+0.357740194 container start d2d48430df7c70c606680d848e4aa5e1c1402c615f99671d9d476ba56a4dcf42 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, vcs-type=git, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, version=8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, com.redhat.component=centos-stream-container, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-02T14:57:44.343 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:44 smithi183 podman[44927]: 2024-04-02 14:57:44.353072819 +0000 UTC m=+0.428935660 container attach d2d48430df7c70c606680d848e4aa5e1c1402c615f99671d9d476ba56a4dcf42 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6-deactivate, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, name=centos-stream, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, io.buildah.version=1.19.8, GIT_CLEAN=True, version=8, ceph=True, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-02T14:57:44.648 INFO:journalctl@ceph.osd.6.smithi183.stdout:Apr 02 14:57:44 smithi183 podman[44927]: 2024-04-02 14:57:44.567595137 +0000 UTC m=+0.643457992 container died d2d48430df7c70c606680d848e4aa5e1c1402c615f99671d9d476ba56a4dcf42 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-6-deactivate, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, com.redhat.component=centos-stream-container, vcs-type=git, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, version=8, ceph=True, name=centos-stream, GIT_BRANCH=HEAD) 2024-04-02T14:57:44.648 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:44 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:44.490+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:44.648 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:44 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:44.490+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:44.648 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:44 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:44.490+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:08.302224+0000 front 2024-04-02T14:57:08.302114+0000 (oldest deadline 2024-04-02T14:57:33.602223+0000) 2024-04-02T14:57:44.649 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:44 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:44.490+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6830 osd.3 since back 2024-04-02T14:57:13.602763+0000 front 2024-04-02T14:57:13.605907+0000 (oldest deadline 2024-04-02T14:57:39.502866+0000) 2024-04-02T14:57:45.103 DEBUG:teuthology.orchestra.run.smithi183:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.6.service' 2024-04-02T14:57:45.143 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T14:57:45.143 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-02T14:57:45.144 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-02T14:57:45.144 DEBUG:teuthology.orchestra.run.smithi183:> sudo systemctl stop ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.7 2024-04-02T14:57:45.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.42 MB/sec execute 71 sec latency 56498.985 ms 2024-04-02T14:57:45.366 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:45 smithi183 systemd[1]: Stopping Ceph osd.7 for bc643104-f0ff-11ee-b647-cb9ed24678a4... 2024-04-02T14:57:45.366 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:45 smithi183 bash[45005]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd.7" found: no such container 2024-04-02T14:57:45.648 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:45 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:45.376+0000 7f4e11bda700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T14:57:45.648 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:45 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:45.376+0000 7f4e11bda700 -1 osd.7 71 *** Got signal Terminated *** 2024-04-02T14:57:45.648 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:45 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:45.376+0000 7f4e11bda700 -1 osd.7 71 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T14:57:45.648 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:45 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:45.500+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:45.648 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:45 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:45.500+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:45.649 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:45 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:45.500+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:08.302224+0000 front 2024-04-02T14:57:08.302114+0000 (oldest deadline 2024-04-02T14:57:33.602223+0000) 2024-04-02T14:57:45.649 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:45 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:45.500+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6830 osd.3 since back 2024-04-02T14:57:13.602763+0000 front 2024-04-02T14:57:13.605907+0000 (oldest deadline 2024-04-02T14:57:39.502866+0000) 2024-04-02T14:57:46.296 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.37 MB/sec execute 72 sec latency 57499.018 ms 2024-04-02T14:57:46.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:46 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:46.465+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:46.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:46 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:46.465+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:46.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:46 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:46.465+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:08.302224+0000 front 2024-04-02T14:57:08.302114+0000 (oldest deadline 2024-04-02T14:57:33.602223+0000) 2024-04-02T14:57:46.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:46 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:46.465+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6830 osd.3 since back 2024-04-02T14:57:13.602763+0000 front 2024-04-02T14:57:13.605907+0000 (oldest deadline 2024-04-02T14:57:39.502866+0000) 2024-04-02T14:57:47.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.33 MB/sec execute 73 sec latency 58499.063 ms 2024-04-02T14:57:47.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:47 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:47.496+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:47.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:47 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:47.496+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:47.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:47 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:47.496+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:08.302224+0000 front 2024-04-02T14:57:08.302114+0000 (oldest deadline 2024-04-02T14:57:33.602223+0000) 2024-04-02T14:57:47.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:47 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:47.496+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6830 osd.3 since back 2024-04-02T14:57:13.602763+0000 front 2024-04-02T14:57:13.605907+0000 (oldest deadline 2024-04-02T14:57:39.502866+0000) 2024-04-02T14:57:48.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.28 MB/sec execute 74 sec latency 59499.090 ms 2024-04-02T14:57:48.799 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:48 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:48.470+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:48.799 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:48 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:48.470+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:48.800 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:48 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:48.470+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:08.302224+0000 front 2024-04-02T14:57:08.302114+0000 (oldest deadline 2024-04-02T14:57:33.602223+0000) 2024-04-02T14:57:48.800 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:48 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:48.470+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6830 osd.3 since back 2024-04-02T14:57:13.602763+0000 front 2024-04-02T14:57:13.605907+0000 (oldest deadline 2024-04-02T14:57:39.502866+0000) 2024-04-02T14:57:49.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.24 MB/sec execute 75 sec latency 60499.130 ms 2024-04-02T14:57:49.898 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:49 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:49.516+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6806 osd.0 since back 2024-04-02T14:56:52.500250+0000 front 2024-04-02T14:56:52.500217+0000 (oldest deadline 2024-04-02T14:57:17.200479+0000) 2024-04-02T14:57:49.913 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:49 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:49.516+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6814 osd.1 since back 2024-04-02T14:56:57.701382+0000 front 2024-04-02T14:56:57.701367+0000 (oldest deadline 2024-04-02T14:57:23.601419+0000) 2024-04-02T14:57:49.913 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:49 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:49.516+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6822 osd.2 since back 2024-04-02T14:57:08.302224+0000 front 2024-04-02T14:57:08.302114+0000 (oldest deadline 2024-04-02T14:57:33.602223+0000) 2024-04-02T14:57:49.913 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:49 smithi183 ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7[40337]: 2024-04-02T14:57:49.516+0000 7f4e0bd70700 -1 osd.7 71 heartbeat_check: no reply from 172.21.15.122:6830 osd.3 since back 2024-04-02T14:57:13.602763+0000 front 2024-04-02T14:57:13.605907+0000 (oldest deadline 2024-04-02T14:57:39.502866+0000) 2024-04-02T14:57:50.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.20 MB/sec execute 76 sec latency 61499.177 ms 2024-04-02T14:57:51.190 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:50 smithi183 podman[45016]: 2024-04-02 14:57:50.925922975 +0000 UTC m=+5.655895959 container died 1406cf2e45a80bb32454706518590898b032bc4d2417d28c5009cd0637f52d25 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, release=754, GIT_CLEAN=True, RELEASE=HEAD, GIT_BRANCH=HEAD, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, architecture=x86_64) 2024-04-02T14:57:51.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.15 MB/sec execute 77 sec latency 62499.206 ms 2024-04-02T14:57:51.557 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:51 smithi183 podman[45016]: 2024-04-02 14:57:51.300002146 +0000 UTC m=+6.029975125 container cleanup 1406cf2e45a80bb32454706518590898b032bc4d2417d28c5009cd0637f52d25 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, distribution-scope=public, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, architecture=x86_64, vendor=Red Hat, Inc., ceph=True, maintainer=Guillaume Abrioux , version=8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, GIT_CLEAN=True, name=centos-stream, build-date=2022-05-03T08:36:31.336870) 2024-04-02T14:57:51.557 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:51 smithi183 bash[45016]: ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7 2024-04-02T14:57:51.558 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:51 smithi183 bash[45058]: Error: no container with name or ID "ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd.7" found: no such container 2024-04-02T14:57:51.847 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:51 smithi183 podman[45045]: 2024-04-02 14:57:51.568274747 +0000 UTC m=+0.642618257 container remove 1406cf2e45a80bb32454706518590898b032bc4d2417d28c5009cd0637f52d25 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.openshift.expose-services=, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, version=8, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, maintainer=Guillaume Abrioux ) 2024-04-02T14:57:51.848 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:51 smithi183 podman[45124]: 2024-04-02 14:57:51.645202126 +0000 UTC m=+0.018508830 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T14:57:51.848 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:51 smithi183 podman[45124]: 2024-04-02 14:57:51.743713862 +0000 UTC m=+0.117020569 container create 87cb27ca3841edb6a9b23d60514c83f3a35cca961c90e9d41962d7a07b79b8f9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7-deactivate, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, vcs-type=git, version=8, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream) 2024-04-02T14:57:52.148 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:51 smithi183 podman[45124]: 2024-04-02 14:57:51.996388167 +0000 UTC m=+0.369694880 container init 87cb27ca3841edb6a9b23d60514c83f3a35cca961c90e9d41962d7a07b79b8f9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7-deactivate, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., vcs-type=git, version=8, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, architecture=x86_64, name=centos-stream) 2024-04-02T14:57:52.148 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:51 smithi183 podman[45124]: 2024-04-02 14:57:51.999782255 +0000 UTC m=+0.373088960 container start 87cb27ca3841edb6a9b23d60514c83f3a35cca961c90e9d41962d7a07b79b8f9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7-deactivate, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, name=centos-stream, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.buildah.version=1.19.8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , vcs-type=git, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc.) 2024-04-02T14:57:52.148 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:52 smithi183 podman[45124]: 2024-04-02 14:57:52.070787229 +0000 UTC m=+0.444093934 container attach 87cb27ca3841edb6a9b23d60514c83f3a35cca961c90e9d41962d7a07b79b8f9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7-deactivate, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, io.openshift.expose-services=, RELEASE=HEAD, ceph=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754) 2024-04-02T14:57:52.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.11 MB/sec execute 78 sec latency 63499.237 ms 2024-04-02T14:57:52.648 INFO:journalctl@ceph.osd.7.smithi183.stdout:Apr 02 14:57:52 smithi183 podman[45124]: 2024-04-02 14:57:52.283165428 +0000 UTC m=+0.656472136 container died 87cb27ca3841edb6a9b23d60514c83f3a35cca961c90e9d41962d7a07b79b8f9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4-osd-7-deactivate, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, ceph=True, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, distribution-scope=public, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, vendor=Red Hat, Inc., io.buildah.version=1.19.8) 2024-04-02T14:57:52.766 DEBUG:teuthology.orchestra.run.smithi183:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bc643104-f0ff-11ee-b647-cb9ed24678a4@osd.7.service' 2024-04-02T14:57:52.809 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T14:57:52.810 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-02T14:57:52.810 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 --force --keep-logs 2024-04-02T14:57:53.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.07 MB/sec execute 79 sec latency 64499.269 ms 2024-04-02T14:57:54.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.04 MB/sec execute 80 sec latency 65499.297 ms 2024-04-02T14:57:55.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 3.00 MB/sec execute 81 sec latency 66499.343 ms 2024-04-02T14:57:56.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.96 MB/sec execute 82 sec latency 67499.373 ms 2024-04-02T14:57:57.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.93 MB/sec execute 83 sec latency 68499.423 ms 2024-04-02T14:57:58.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.89 MB/sec execute 84 sec latency 69499.451 ms 2024-04-02T14:57:59.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.86 MB/sec execute 85 sec latency 70499.827 ms 2024-04-02T14:58:00.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.82 MB/sec execute 86 sec latency 71499.856 ms 2024-04-02T14:58:01.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.79 MB/sec execute 87 sec latency 72499.884 ms 2024-04-02T14:58:02.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.76 MB/sec execute 88 sec latency 73499.922 ms 2024-04-02T14:58:03.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.73 MB/sec execute 89 sec latency 74499.939 ms 2024-04-02T14:58:04.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.70 MB/sec execute 90 sec latency 75499.954 ms 2024-04-02T14:58:05.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.67 MB/sec execute 91 sec latency 76499.971 ms 2024-04-02T14:58:06.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.64 MB/sec execute 92 sec latency 77499.987 ms 2024-04-02T14:58:07.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.61 MB/sec execute 93 sec latency 78500.013 ms 2024-04-02T14:58:08.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.58 MB/sec execute 94 sec latency 79500.028 ms 2024-04-02T14:58:09.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.56 MB/sec execute 95 sec latency 80500.045 ms 2024-04-02T14:58:10.297 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.53 MB/sec execute 96 sec latency 81500.082 ms 2024-04-02T14:58:11.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.50 MB/sec execute 97 sec latency 82500.111 ms 2024-04-02T14:58:12.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.48 MB/sec execute 98 sec latency 83500.193 ms 2024-04-02T14:58:13.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.45 MB/sec execute 99 sec latency 84500.239 ms 2024-04-02T14:58:14.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.43 MB/sec execute 100 sec latency 85500.273 ms 2024-04-02T14:58:15.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.40 MB/sec execute 101 sec latency 86500.310 ms 2024-04-02T14:58:16.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.38 MB/sec execute 102 sec latency 87500.348 ms 2024-04-02T14:58:17.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.36 MB/sec execute 103 sec latency 88500.385 ms 2024-04-02T14:58:18.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.34 MB/sec execute 104 sec latency 89500.422 ms 2024-04-02T14:58:19.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.31 MB/sec execute 105 sec latency 90500.463 ms 2024-04-02T14:58:20.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.29 MB/sec execute 106 sec latency 91500.509 ms 2024-04-02T14:58:21.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.27 MB/sec execute 107 sec latency 92500.556 ms 2024-04-02T14:58:22.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.25 MB/sec execute 108 sec latency 93500.646 ms 2024-04-02T14:58:23.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.23 MB/sec execute 109 sec latency 94500.691 ms 2024-04-02T14:58:23.325 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 --force --keep-logs 2024-04-02T14:58:24.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.21 MB/sec execute 110 sec latency 95500.790 ms 2024-04-02T14:58:25.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.19 MB/sec execute 111 sec latency 96500.878 ms 2024-04-02T14:58:26.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.17 MB/sec execute 112 sec latency 97500.981 ms 2024-04-02T14:58:27.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.15 MB/sec execute 113 sec latency 98501.068 ms 2024-04-02T14:58:28.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.13 MB/sec execute 114 sec latency 99501.128 ms 2024-04-02T14:58:29.298 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.11 MB/sec execute 115 sec latency 100501.207 ms 2024-04-02T14:58:30.299 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.09 MB/sec execute 116 sec latency 101501.282 ms 2024-04-02T14:58:31.299 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.08 MB/sec execute 117 sec latency 102501.397 ms 2024-04-02T14:58:32.299 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.06 MB/sec execute 118 sec latency 103501.455 ms 2024-04-02T14:58:33.299 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.04 MB/sec execute 119 sec latency 104501.543 ms 2024-04-02T14:58:34.299 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.02 MB/sec execute 120 sec latency 105501.646 ms 2024-04-02T14:58:35.299 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 2.01 MB/sec execute 121 sec latency 106501.758 ms 2024-04-02T14:58:36.299 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.99 MB/sec execute 122 sec latency 107501.852 ms 2024-04-02T14:58:37.299 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.97 MB/sec execute 123 sec latency 108501.929 ms 2024-04-02T14:58:38.299 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.96 MB/sec execute 124 sec latency 109501.998 ms 2024-04-02T14:58:39.299 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.94 MB/sec execute 125 sec latency 110502.062 ms 2024-04-02T14:58:40.300 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.93 MB/sec execute 126 sec latency 111502.174 ms 2024-04-02T14:58:41.300 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.91 MB/sec execute 127 sec latency 112502.256 ms 2024-04-02T14:58:42.127 DEBUG:teuthology.orchestra.run.smithi122:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-02T14:58:42.162 DEBUG:teuthology.orchestra.run.smithi183:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-02T14:58:42.194 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-02T14:58:42.196 DEBUG:teuthology.misc:Transferring archived files from smithi122:/var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/crash to /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636013/remote/smithi122/crash 2024-04-02T14:58:42.197 DEBUG:teuthology.orchestra.run.smithi122:> sudo tar c -f - -C /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/crash -- . 2024-04-02T14:58:42.248 INFO:teuthology.orchestra.run.smithi122.stderr:tar: /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-04-02T14:58:42.248 INFO:teuthology.orchestra.run.smithi122.stderr:tar: Error is not recoverable: exiting now 2024-04-02T14:58:42.250 DEBUG:teuthology.misc:Transferring archived files from smithi183:/var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/crash to /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636013/remote/smithi183/crash 2024-04-02T14:58:42.251 DEBUG:teuthology.orchestra.run.smithi183:> sudo tar c -f - -C /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/crash -- . 2024-04-02T14:58:42.300 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.90 MB/sec execute 128 sec latency 113502.352 ms 2024-04-02T14:58:42.313 INFO:teuthology.orchestra.run.smithi183.stderr:tar: /var/lib/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-04-02T14:58:42.313 INFO:teuthology.orchestra.run.smithi183.stderr:tar: Error is not recoverable: exiting now 2024-04-02T14:58:42.315 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-02T14:58:42.315 DEBUG:teuthology.orchestra.run.smithi122:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-02T14:58:42.346 INFO:teuthology.orchestra.run.smithi122.stdout:2024-04-02T14:51:55.551593+0000 mon.smithi122 (mon.0) 504 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-02T14:58:42.347 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-02T14:58:42.347 INFO:tasks.cephadm:Compressing logs... 2024-04-02T14:58:42.347 DEBUG:teuthology.orchestra.run.smithi122:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-02T14:58:42.389 DEBUG:teuthology.orchestra.run.smithi183:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-02T14:58:42.425 INFO:teuthology.orchestra.run.smithi122.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-02T14:58:42.425 INFO:teuthology.orchestra.run.smithi122.stderr:: No such file or directory 2024-04-02T14:58:42.426 INFO:teuthology.orchestra.run.smithi122.stderr:gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-mon.smithi122.log 2024-04-02T14:58:42.426 INFO:teuthology.orchestra.run.smithi122.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-mds.foofs.smithi122.cghwpx.log 2024-04-02T14:58:42.426 INFO:teuthology.orchestra.run.smithi122.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-mon.smithi122.log: gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph.audit.log 2024-04-02T14:58:42.426 INFO:teuthology.orchestra.run.smithi122.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-mds.foofs.smithi122.cghwpx.log: gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.1.log 2024-04-02T14:58:42.426 INFO:teuthology.orchestra.run.smithi122.stderr: 81.5% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-mds.foofs.smithi122.cghwpx.log.gz 2024-04-02T14:58:42.426 INFO:teuthology.orchestra.run.smithi122.stderr:gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.3.log 2024-04-02T14:58:42.426 INFO:teuthology.orchestra.run.smithi122.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph.audit.log: /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-mgr.smithi122.szjkfg.log 2024-04-02T14:58:42.426 INFO:teuthology.orchestra.run.smithi122.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.0.log 2024-04-02T14:58:42.427 INFO:teuthology.orchestra.run.smithi122.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-mgr.smithi122.szjkfg.log: gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph.cephadm.log 2024-04-02T14:58:42.427 INFO:teuthology.orchestra.run.smithi122.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph.log 2024-04-02T14:58:42.427 INFO:teuthology.orchestra.run.smithi122.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph.cephadm.log: 90.8% 83.0% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph.cephadm.log.gz 2024-04-02T14:58:42.427 INFO:teuthology.orchestra.run.smithi122.stderr: -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph.audit.log.gz 2024-04-02T14:58:42.427 INFO:teuthology.orchestra.run.smithi122.stderr:gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.2.log 2024-04-02T14:58:42.427 INFO:teuthology.orchestra.run.smithi122.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph.log: gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-volume.log 2024-04-02T14:58:42.427 INFO:teuthology.orchestra.run.smithi122.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.2.log: 86.2% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph.log.gz 2024-04-02T14:58:42.428 INFO:teuthology.orchestra.run.smithi183.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-02T14:58:42.428 INFO:teuthology.orchestra.run.smithi183.stderr:: No such file or directory 2024-04-02T14:58:42.429 INFO:teuthology.orchestra.run.smithi183.stderr:gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.6.log 2024-04-02T14:58:42.429 INFO:teuthology.orchestra.run.smithi183.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-mds.foofs.smithi183.idnham.log 2024-04-02T14:58:42.429 INFO:teuthology.orchestra.run.smithi183.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-mon.smithi183.log 2024-04-02T14:58:42.429 INFO:teuthology.orchestra.run.smithi183.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-mds.foofs.smithi183.idnham.log: gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph.audit.log 2024-04-02T14:58:42.429 INFO:teuthology.orchestra.run.smithi183.stderr: 60.7% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-mds.foofs.smithi183.idnham.log.gz 2024-04-02T14:58:42.429 INFO:teuthology.orchestra.run.smithi183.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-mon.smithi183.log: gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.4.log 2024-04-02T14:58:42.429 INFO:teuthology.orchestra.run.smithi183.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.7.log 2024-04-02T14:58:42.429 INFO:teuthology.orchestra.run.smithi183.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.5.log 2024-04-02T14:58:42.429 INFO:teuthology.orchestra.run.smithi183.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph.cephadm.log 2024-04-02T14:58:42.430 INFO:teuthology.orchestra.run.smithi183.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-mgr.smithi183.smxedq.log 2024-04-02T14:58:42.430 INFO:teuthology.orchestra.run.smithi183.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph.cephadm.log: 91.0% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph.audit.log.gz 2024-04-02T14:58:42.430 INFO:teuthology.orchestra.run.smithi183.stderr: 82.1% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph.cephadm.log.gz 2024-04-02T14:58:42.430 INFO:teuthology.orchestra.run.smithi183.stderr:gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph.log 2024-04-02T14:58:42.430 INFO:teuthology.orchestra.run.smithi183.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-mgr.smithi183.smxedq.log: gzip -5 --verbose -- /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-volume.log 2024-04-02T14:58:42.430 INFO:teuthology.orchestra.run.smithi183.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph.log: 86.1% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph.log.gz 2024-04-02T14:58:42.430 INFO:teuthology.orchestra.run.smithi183.stderr: 90.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-02T14:58:42.431 INFO:teuthology.orchestra.run.smithi183.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-volume.log: 91.1% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-volume.log.gz 2024-04-02T14:58:42.431 INFO:teuthology.orchestra.run.smithi122.stderr: 92.0% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-02T14:58:42.434 INFO:teuthology.orchestra.run.smithi183.stderr: 91.7% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-mgr.smithi183.smxedq.log.gz 2024-04-02T14:58:42.437 INFO:teuthology.orchestra.run.smithi122.stderr:/var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-volume.log: 91.1% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-volume.log.gz 2024-04-02T14:58:42.522 INFO:teuthology.orchestra.run.smithi183.stderr: 92.6% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-mon.smithi183.log.gz 2024-04-02T14:58:42.567 INFO:teuthology.orchestra.run.smithi122.stderr: 88.4% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-mgr.smithi122.szjkfg.log.gz 2024-04-02T14:58:43.062 INFO:teuthology.orchestra.run.smithi122.stderr: 91.9% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-mon.smithi122.log.gz 2024-04-02T14:58:43.300 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.88 MB/sec execute 129 sec latency 114502.386 ms 2024-04-02T14:58:44.300 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.87 MB/sec execute 130 sec latency 115502.428 ms 2024-04-02T14:58:45.221 INFO:teuthology.orchestra.run.smithi122.stderr: 94.6% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.2.log.gz 2024-04-02T14:58:45.300 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.85 MB/sec execute 131 sec latency 116502.468 ms 2024-04-02T14:58:46.226 INFO:teuthology.orchestra.run.smithi183.stderr: 94.1% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.4.log.gz 2024-04-02T14:58:46.300 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.84 MB/sec execute 132 sec latency 117502.505 ms 2024-04-02T14:58:46.475 INFO:teuthology.orchestra.run.smithi183.stderr: 94.4% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.6.log.gz 2024-04-02T14:58:46.631 INFO:teuthology.orchestra.run.smithi122.stderr: 94.1% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.3.log.gz 2024-04-02T14:58:46.949 INFO:teuthology.orchestra.run.smithi122.stderr: 94.2% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.0.log.gz 2024-04-02T14:58:46.956 INFO:teuthology.orchestra.run.smithi122.stderr: 94.3% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.1.log.gz 2024-04-02T14:58:46.958 INFO:teuthology.orchestra.run.smithi122.stderr: 2024-04-02T14:58:46.958 INFO:teuthology.orchestra.run.smithi122.stderr:real 0m4.556s 2024-04-02T14:58:46.958 INFO:teuthology.orchestra.run.smithi122.stderr:user 0m16.268s 2024-04-02T14:58:46.959 INFO:teuthology.orchestra.run.smithi122.stderr:sys 0m0.639s 2024-04-02T14:58:47.300 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.83 MB/sec execute 133 sec latency 118502.614 ms 2024-04-02T14:58:47.482 INFO:teuthology.orchestra.run.smithi183.stderr: 94.3% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.5.log.gz 2024-04-02T14:58:48.300 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.81 MB/sec execute 134 sec latency 119502.697 ms 2024-04-02T14:58:48.654 INFO:teuthology.orchestra.run.smithi183.stderr: 94.2% -- replaced with /var/log/ceph/bc643104-f0ff-11ee-b647-cb9ed24678a4/ceph-osd.7.log.gz 2024-04-02T14:58:48.656 INFO:teuthology.orchestra.run.smithi183.stderr: 2024-04-02T14:58:48.656 INFO:teuthology.orchestra.run.smithi183.stderr:real 0m6.253s 2024-04-02T14:58:48.657 INFO:teuthology.orchestra.run.smithi183.stderr:user 0m18.534s 2024-04-02T14:58:48.657 INFO:teuthology.orchestra.run.smithi183.stderr:sys 0m0.767s 2024-04-02T14:58:48.657 INFO:tasks.cephadm:Archiving logs... 2024-04-02T14:58:48.657 DEBUG:teuthology.misc:Transferring archived files from smithi122:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636013/remote/smithi122/log 2024-04-02T14:58:48.658 DEBUG:teuthology.orchestra.run.smithi122:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-02T14:58:49.319 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.80 MB/sec execute 135 sec latency 120502.809 ms 2024-04-02T14:58:50.314 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.79 MB/sec execute 136 sec latency 121502.866 ms 2024-04-02T14:58:50.977 DEBUG:teuthology.misc:Transferring archived files from smithi183:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636013/remote/smithi183/log 2024-04-02T14:58:50.978 DEBUG:teuthology.orchestra.run.smithi183:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-02T14:58:51.312 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.77 MB/sec execute 137 sec latency 122502.971 ms 2024-04-02T14:58:52.318 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.76 MB/sec execute 138 sec latency 123503.055 ms 2024-04-02T14:58:53.305 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.75 MB/sec execute 139 sec latency 124503.166 ms 2024-04-02T14:58:53.723 INFO:tasks.cephadm:Removing cluster... 2024-04-02T14:58:53.724 DEBUG:teuthology.orchestra.run.smithi122:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 --force 2024-04-02T14:58:54.029 DEBUG:teuthology.orchestra.run.smithi183:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 --force 2024-04-02T14:58:54.300 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.73 MB/sec execute 140 sec latency 125503.234 ms 2024-04-02T14:58:54.348 INFO:tasks.cephadm:Removing cephadm ... 2024-04-02T14:58:54.349 DEBUG:teuthology.orchestra.run.smithi122:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-02T14:58:54.364 DEBUG:teuthology.orchestra.run.smithi183:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-02T14:58:54.379 INFO:tasks.cephadm:Teardown complete 2024-04-02T14:58:54.379 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-02T14:58:54.389 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-02T14:58:54.389 DEBUG:teuthology.orchestra.run.smithi122:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-02T14:58:54.406 DEBUG:teuthology.orchestra.run.smithi183:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-02T14:58:54.418 INFO:teuthology.orchestra.run.smithi122.stderr:bash: line 1: ntpq: command not found 2024-04-02T14:58:54.422 INFO:teuthology.orchestra.run.smithi122.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-02T14:58:54.422 INFO:teuthology.orchestra.run.smithi122.stdout:=============================================================================== 2024-04-02T14:58:54.423 INFO:teuthology.orchestra.run.smithi122.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 49 -592us[ -592us] +/- 93ms 2024-04-02T14:58:54.423 INFO:teuthology.orchestra.run.smithi122.stdout:^* hv02.front.sepia.ceph.com 3 6 377 51 -1248us[-1281us] +/- 82ms 2024-04-02T14:58:54.423 INFO:teuthology.orchestra.run.smithi122.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 52 +1622us[+1589us] +/- 71ms 2024-04-02T14:58:54.423 INFO:teuthology.orchestra.run.smithi122.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T14:58:54.433 INFO:teuthology.orchestra.run.smithi183.stderr:bash: line 1: ntpq: command not found 2024-04-02T14:58:54.437 INFO:teuthology.orchestra.run.smithi183.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-02T14:58:54.437 INFO:teuthology.orchestra.run.smithi183.stdout:=============================================================================== 2024-04-02T14:58:54.437 INFO:teuthology.orchestra.run.smithi183.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 48 -609us[ -609us] +/- 93ms 2024-04-02T14:58:54.437 INFO:teuthology.orchestra.run.smithi183.stdout:^* hv02.front.sepia.ceph.com 3 6 377 49 -1246us[-1259us] +/- 82ms 2024-04-02T14:58:54.437 INFO:teuthology.orchestra.run.smithi183.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 49 +1600us[+1600us] +/- 72ms 2024-04-02T14:58:54.437 INFO:teuthology.orchestra.run.smithi183.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T14:58:54.438 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-02T14:58:54.447 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-02T14:58:54.448 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-02T14:58:54.470 DEBUG:teuthology.orchestra.run.smithi122:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-02T14:58:54.543 DEBUG:teuthology.orchestra.run.smithi183:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-02T14:58:54.614 DEBUG:teuthology.orchestra.run.smithi122:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-02T14:58:54.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T14:58:54.644 DEBUG:teuthology.orchestra.run.smithi183:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-02T14:58:54.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T14:58:54.674 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-02T14:58:54.683 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-02T14:58:54.728 INFO:teuthology.task.internal:Duration was 1028.171919 seconds 2024-04-02T14:58:54.729 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-02T14:58:54.771 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-02T14:58:54.771 DEBUG:teuthology.orchestra.run.smithi122:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-02T14:58:54.774 DEBUG:teuthology.orchestra.run.smithi183:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-02T14:58:54.815 INFO:teuthology.orchestra.run.smithi122.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-02T14:58:54.818 INFO:teuthology.orchestra.run.smithi183.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-02T14:58:55.129 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-02T14:58:55.129 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi122.front.sepia.ceph.com 2024-04-02T14:58:55.129 DEBUG:teuthology.orchestra.run.smithi122:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-02T14:58:55.190 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi183.front.sepia.ceph.com 2024-04-02T14:58:55.191 DEBUG:teuthology.orchestra.run.smithi183:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-02T14:58:55.219 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-02T14:58:55.219 DEBUG:teuthology.orchestra.run.smithi122:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-02T14:58:55.235 DEBUG:teuthology.orchestra.run.smithi183:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-02T14:58:55.287 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-02T14:58:55.288 DEBUG:teuthology.orchestra.run.smithi122:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-02T14:58:55.300 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.72 MB/sec execute 141 sec latency 126503.339 ms 2024-04-02T14:58:55.302 DEBUG:teuthology.orchestra.run.smithi183:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-02T14:58:56.301 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.71 MB/sec execute 142 sec latency 127503.381 ms 2024-04-02T14:58:57.301 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.70 MB/sec execute 143 sec latency 128503.432 ms 2024-04-02T14:58:58.301 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.69 MB/sec execute 144 sec latency 129503.482 ms 2024-04-02T14:58:59.301 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.68 MB/sec execute 145 sec latency 130503.537 ms 2024-04-02T14:58:59.710 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-02T14:58:59.721 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-02T14:58:59.721 DEBUG:teuthology.orchestra.run.smithi122:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-02T14:58:59.748 DEBUG:teuthology.orchestra.run.smithi183:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-02T14:58:59.781 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-02T14:58:59.792 DEBUG:teuthology.orchestra.run.smithi122:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-02T14:58:59.795 DEBUG:teuthology.orchestra.run.smithi183:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-02T14:58:59.818 INFO:teuthology.orchestra.run.smithi122.stdout:kernel.core_pattern = core 2024-04-02T14:58:59.851 INFO:teuthology.orchestra.run.smithi183.stdout:kernel.core_pattern = core 2024-04-02T14:58:59.879 DEBUG:teuthology.orchestra.run.smithi122:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-02T14:58:59.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T14:58:59.896 DEBUG:teuthology.orchestra.run.smithi183:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-02T14:58:59.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T14:58:59.935 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-02T14:58:59.945 INFO:teuthology.task.internal:Transferring archived files... 2024-04-02T14:58:59.946 DEBUG:teuthology.misc:Transferring archived files from smithi122:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636013/remote/smithi122 2024-04-02T14:58:59.946 DEBUG:teuthology.orchestra.run.smithi122:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-02T14:58:59.994 DEBUG:teuthology.misc:Transferring archived files from smithi183:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636013/remote/smithi183 2024-04-02T14:58:59.995 DEBUG:teuthology.orchestra.run.smithi183:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-02T14:59:00.036 INFO:teuthology.task.internal:Removing archive directory... 2024-04-02T14:59:00.036 DEBUG:teuthology.orchestra.run.smithi122:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-02T14:59:00.039 DEBUG:teuthology.orchestra.run.smithi183:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-02T14:59:00.086 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-02T14:59:00.098 INFO:teuthology.task.internal:Not uploading archives. 2024-04-02T14:59:00.098 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-02T14:59:00.134 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-02T14:59:00.134 DEBUG:teuthology.orchestra.run.smithi122:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-02T14:59:00.137 DEBUG:teuthology.orchestra.run.smithi183:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-02T14:59:00.149 INFO:teuthology.orchestra.run.smithi122.stdout: 265594 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 2 14:59 /home/ubuntu/cephtest 2024-04-02T14:59:00.152 INFO:teuthology.orchestra.run.smithi183.stdout: 266042 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 2 14:59 /home/ubuntu/cephtest 2024-04-02T14:59:00.153 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-02T14:59:00.185 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-02T14:59:00.254 INFO:teuthology.nuke:Checking targets against current locks 2024-04-02T14:59:00.281 DEBUG:teuthology.nuke:shortname: smithi122 2024-04-02T14:59:00.282 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-02T14:59:00.301 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.66 MB/sec execute 146 sec latency 131503.665 ms 2024-04-02T14:59:00.302 DEBUG:teuthology.nuke:shortname: smithi183 2024-04-02T14:59:00.302 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-02T14:59:00.307 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi122.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636013', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-02 14:33:40.640965', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGn0D8IanrmG7WQtzHNx+XSHVkFRT/s6VVCHg/pgMOUE28lRsj0l+gwINiAGP0/yZFazAstH4mlJUkLXBhLqPcI='} 2024-04-02T14:59:00.322 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi183.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636013', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-02 14:33:40.639385', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK2vGxYt4xuDcnKqESTnYfqv10YcFkW7XasL+xY1XG+Aa36VQ2XK8sNyNWA7mTDzo6LScCKAv57qmOCoJJ1He0o='} 2024-04-02T14:59:00.353 INFO:teuthology.orchestra.console.smithi122:Power off 2024-04-02T14:59:00.353 DEBUG:teuthology.orchestra.console.smithi122:pexpect command: ipmitool -H smithi122.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-02T14:59:00.368 INFO:teuthology.orchestra.console.smithi183:Power off 2024-04-02T14:59:00.369 DEBUG:teuthology.orchestra.console.smithi183:pexpect command: ipmitool -H smithi183.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-02T14:59:00.377 DEBUG:teuthology.orchestra.console.smithi122:power off output: Chassis Power Control: Down/Off 2024-04-02T14:59:00.377 DEBUG:teuthology.orchestra.console.smithi122:pexpect command: ipmitool -H smithi122.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T14:59:00.390 DEBUG:teuthology.orchestra.console.smithi183:power off output: Chassis Power Control: Down/Off 2024-04-02T14:59:00.391 DEBUG:teuthology.orchestra.console.smithi183:pexpect command: ipmitool -H smithi183.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T14:59:00.400 DEBUG:teuthology.orchestra.console.smithi122:check power output: Chassis Power is on 2024-04-02T14:59:00.433 DEBUG:teuthology.orchestra.console.smithi183:check power output: Chassis Power is on 2024-04-02T14:59:01.301 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.65 MB/sec execute 147 sec latency 132503.739 ms 2024-04-02T14:59:02.301 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.64 MB/sec execute 148 sec latency 133503.840 ms 2024-04-02T14:59:03.301 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.63 MB/sec execute 149 sec latency 134503.912 ms 2024-04-02T14:59:04.301 INFO:teuthology.orchestra.run.smithi122.stdout: 5 78983 1.62 MB/sec execute 150 sec latency 135503.933 ms 2024-04-02T14:59:04.331 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T14:59:04.401 DEBUG:teuthology.orchestra.console.smithi122:pexpect command: ipmitool -H smithi122.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T14:59:04.434 DEBUG:teuthology.orchestra.console.smithi183:pexpect command: ipmitool -H smithi183.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T14:59:04.511 DEBUG:teuthology.orchestra.console.smithi122:check power output: Chassis Power is on 2024-04-02T14:59:04.543 DEBUG:teuthology.orchestra.console.smithi183:check power output: Chassis Power is on 2024-04-02T14:59:08.514 DEBUG:teuthology.orchestra.console.smithi122:pexpect command: ipmitool -H smithi122.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T14:59:08.544 DEBUG:teuthology.orchestra.console.smithi183:pexpect command: ipmitool -H smithi183.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T14:59:08.625 DEBUG:teuthology.orchestra.console.smithi122:check power output: Chassis Power is on 2024-04-02T14:59:08.654 DEBUG:teuthology.orchestra.console.smithi183:check power output: Chassis Power is off 2024-04-02T14:59:08.754 INFO:teuthology.orchestra.console.smithi183:Power off completed 2024-04-02T14:59:12.626 DEBUG:teuthology.orchestra.console.smithi122:pexpect command: ipmitool -H smithi122.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T14:59:12.736 DEBUG:teuthology.orchestra.console.smithi122:check power output: Chassis Power is off 2024-04-02T14:59:12.837 INFO:teuthology.orchestra.console.smithi122:Power off completed 2024-04-02T14:59:12.941 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1028.1719188690186 failure_reason: 'Command failed on smithi122 with status 22: "sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bc643104-f0ff-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c ''ceph orch ps''"' owner: scheduled_gabrioux@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=7c9369ba58ea4da69557a7243ac5e8bf status: fail success: false 2024-04-02T14:59:12.941 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-02T14:59:13.010 INFO:teuthology.run:FAIL