2024-04-15T21:19:22.990 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-15T21:19:22.990 DEBUG:teuthology.run:Teuthology command: teuthology --description orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} --verbose --archive /home/teuthworker/archive/teuthology-2024-04-15_21:08:03-orch-squid-distro-default-smithi/7657448 --name teuthology-2024-04-15_21:08:03-orch-squid-distro-default-smithi --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2024-04-15_21:08:03-orch-squid-distro-default-smithi/7657448/orig.config.yaml 2024-04-15T21:19:23.027 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-15T21:19:23.125 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-15_21:08:03-orch-squid-distro-default-smithi/7657448 branch: squid description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: ceph-qa@ceph.com first_in_suite: false job_id: '7657448' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-04-15_21:08:03-orch-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: a9a752df26c63acad72e1b3569fd79a515ca0765 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: a9a752df26c63acad72e1b3569fd79a515ca0765 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: squid sha1: 461a84e6fbbf089011eafd13b98169fb9e9b8cc9 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 7775 sha1: a9a752df26c63acad72e1b3569fd79a515ca0765 sleep_before_teardown: 0 subset: 18/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_461a84e6fbbf089011eafd13b98169fb9e9b8cc9/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 461a84e6fbbf089011eafd13b98169fb9e9b8cc9 targets: smithi083.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNUbAozp6fZ0b5gIPjYHw29AUj/hOYr3KTuGHNYVL0yOOb3LB5Z73TwEcdvFlDV6p9VadW2GtPIhl8yTfj1gvdQ= smithi139.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMQn2Rjol8cML9HuaQ7TgtO3uAw2/xk0vArkIKOQNNOm/E7SdbgQkyl8W+3uIU/5V98z1ar54u8GyjH0fx3J120= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-15_21:08:03 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-04-15T21:19:23.125 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_461a84e6fbbf089011eafd13b98169fb9e9b8cc9/qa; will attempt to use it 2024-04-15T21:19:23.126 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_461a84e6fbbf089011eafd13b98169fb9e9b8cc9/qa/tasks 2024-04-15T21:19:23.126 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-15T21:19:23.128 INFO:teuthology.task.internal:Checking packages... 2024-04-15T21:19:23.152 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'a9a752df26c63acad72e1b3569fd79a515ca0765' 2024-04-15T21:19:23.152 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-15T21:19:23.153 INFO:teuthology.packaging:ref: None 2024-04-15T21:19:23.153 INFO:teuthology.packaging:tag: None 2024-04-15T21:19:23.153 INFO:teuthology.packaging:branch: squid 2024-04-15T21:19:23.153 INFO:teuthology.packaging:sha1: a9a752df26c63acad72e1b3569fd79a515ca0765 2024-04-15T21:19:23.153 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-04-15T21:19:23.414 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2114.ga9a752df 2024-04-15T21:19:23.416 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-15T21:19:23.423 INFO:teuthology.task.internal:no buildpackages task found 2024-04-15T21:19:23.423 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-15T21:19:23.430 INFO:teuthology.task.internal:Saving configuration 2024-04-15T21:19:23.444 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-15T21:19:23.450 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-15T21:19:23.476 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi083.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-15_21:08:03-orch-squid-distro-default-smithi/7657448', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-15 21:14:35.079515', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNUbAozp6fZ0b5gIPjYHw29AUj/hOYr3KTuGHNYVL0yOOb3LB5Z73TwEcdvFlDV6p9VadW2GtPIhl8yTfj1gvdQ='} 2024-04-15T21:19:23.499 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi139.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-15_21:08:03-orch-squid-distro-default-smithi/7657448', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-15 21:14:35.081276', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMQn2Rjol8cML9HuaQ7TgtO3uAw2/xk0vArkIKOQNNOm/E7SdbgQkyl8W+3uIU/5V98z1ar54u8GyjH0fx3J120='} 2024-04-15T21:19:23.499 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-15T21:19:23.505 INFO:teuthology.task.internal:roles: ubuntu@smithi083.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-04-15T21:19:23.505 INFO:teuthology.task.internal:roles: ubuntu@smithi139.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-04-15T21:19:23.505 INFO:teuthology.run_tasks:Running task console_log... 2024-04-15T21:19:23.583 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fbc7df95f70>, signals=[15]) 2024-04-15T21:19:23.583 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-15T21:19:23.589 INFO:teuthology.task.internal:Opening connections... 2024-04-15T21:19:23.589 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi083.front.sepia.ceph.com 2024-04-15T21:19:23.591 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-15T21:19:23.683 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi139.front.sepia.ceph.com 2024-04-15T21:19:23.684 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi139.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-15T21:19:23.771 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-15T21:19:23.779 DEBUG:teuthology.orchestra.run.smithi083:> uname -m 2024-04-15T21:19:23.797 INFO:teuthology.orchestra.run.smithi083.stdout:x86_64 2024-04-15T21:19:23.797 DEBUG:teuthology.orchestra.run.smithi083:> cat /etc/os-release 2024-04-15T21:19:23.854 INFO:teuthology.orchestra.run.smithi083.stdout:NAME="CentOS Stream" 2024-04-15T21:19:23.854 INFO:teuthology.orchestra.run.smithi083.stdout:VERSION="9" 2024-04-15T21:19:23.854 INFO:teuthology.orchestra.run.smithi083.stdout:ID="centos" 2024-04-15T21:19:23.854 INFO:teuthology.orchestra.run.smithi083.stdout:ID_LIKE="rhel fedora" 2024-04-15T21:19:23.854 INFO:teuthology.orchestra.run.smithi083.stdout:VERSION_ID="9" 2024-04-15T21:19:23.854 INFO:teuthology.orchestra.run.smithi083.stdout:PLATFORM_ID="platform:el9" 2024-04-15T21:19:23.854 INFO:teuthology.orchestra.run.smithi083.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-15T21:19:23.854 INFO:teuthology.orchestra.run.smithi083.stdout:ANSI_COLOR="0;31" 2024-04-15T21:19:23.855 INFO:teuthology.orchestra.run.smithi083.stdout:LOGO="fedora-logo-icon" 2024-04-15T21:19:23.855 INFO:teuthology.orchestra.run.smithi083.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-15T21:19:23.855 INFO:teuthology.orchestra.run.smithi083.stdout:HOME_URL="https://centos.org/" 2024-04-15T21:19:23.855 INFO:teuthology.orchestra.run.smithi083.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-15T21:19:23.855 INFO:teuthology.orchestra.run.smithi083.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-15T21:19:23.855 INFO:teuthology.orchestra.run.smithi083.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-15T21:19:23.856 INFO:teuthology.lock.ops:Updating smithi083.front.sepia.ceph.com on lock server 2024-04-15T21:19:23.875 DEBUG:teuthology.orchestra.run.smithi139:> uname -m 2024-04-15T21:19:23.892 INFO:teuthology.orchestra.run.smithi139.stdout:x86_64 2024-04-15T21:19:23.893 DEBUG:teuthology.orchestra.run.smithi139:> cat /etc/os-release 2024-04-15T21:19:23.949 INFO:teuthology.orchestra.run.smithi139.stdout:NAME="CentOS Stream" 2024-04-15T21:19:23.950 INFO:teuthology.orchestra.run.smithi139.stdout:VERSION="9" 2024-04-15T21:19:23.950 INFO:teuthology.orchestra.run.smithi139.stdout:ID="centos" 2024-04-15T21:19:23.950 INFO:teuthology.orchestra.run.smithi139.stdout:ID_LIKE="rhel fedora" 2024-04-15T21:19:23.950 INFO:teuthology.orchestra.run.smithi139.stdout:VERSION_ID="9" 2024-04-15T21:19:23.950 INFO:teuthology.orchestra.run.smithi139.stdout:PLATFORM_ID="platform:el9" 2024-04-15T21:19:23.950 INFO:teuthology.orchestra.run.smithi139.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-15T21:19:23.950 INFO:teuthology.orchestra.run.smithi139.stdout:ANSI_COLOR="0;31" 2024-04-15T21:19:23.950 INFO:teuthology.orchestra.run.smithi139.stdout:LOGO="fedora-logo-icon" 2024-04-15T21:19:23.950 INFO:teuthology.orchestra.run.smithi139.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-15T21:19:23.950 INFO:teuthology.orchestra.run.smithi139.stdout:HOME_URL="https://centos.org/" 2024-04-15T21:19:23.950 INFO:teuthology.orchestra.run.smithi139.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-15T21:19:23.950 INFO:teuthology.orchestra.run.smithi139.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-15T21:19:23.951 INFO:teuthology.orchestra.run.smithi139.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-15T21:19:23.951 INFO:teuthology.lock.ops:Updating smithi139.front.sepia.ceph.com on lock server 2024-04-15T21:19:23.975 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-15T21:19:23.984 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-15T21:19:23.993 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-15T21:19:23.993 DEBUG:teuthology.orchestra.run.smithi083:> test '!' -e /home/ubuntu/cephtest 2024-04-15T21:19:23.995 DEBUG:teuthology.orchestra.run.smithi139:> test '!' -e /home/ubuntu/cephtest 2024-04-15T21:19:24.012 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-15T21:19:24.017 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-15T21:19:24.017 DEBUG:teuthology.orchestra.run.smithi083:> test -z $(ls -A /var/lib/ceph) 2024-04-15T21:19:24.051 DEBUG:teuthology.orchestra.run.smithi139:> test -z $(ls -A /var/lib/ceph) 2024-04-15T21:19:24.159 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-15T21:19:24.215 INFO:teuthology.run_tasks:Running task kernel... 2024-04-15T21:19:24.230 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-15T21:19:24.230 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-15T21:19:24.230 DEBUG:teuthology.orchestra.run.smithi083:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-15T21:19:24.231 DEBUG:teuthology.orchestra.run.smithi139:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-15T21:19:24.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-15T21:19:24.247 DEBUG:teuthology.orchestra.run.smithi083:> uname -r 2024-04-15T21:19:24.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-15T21:19:24.248 DEBUG:teuthology.orchestra.run.smithi139:> uname -r 2024-04-15T21:19:24.304 INFO:teuthology.orchestra.run.smithi083.stdout:5.14.0-361.el9.x86_64 2024-04-15T21:19:24.304 INFO:teuthology.task.kernel:Running kernel on smithi083: 5.14.0-361.el9.x86_64 2024-04-15T21:19:24.304 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum install -y kernel 2024-04-15T21:19:24.305 INFO:teuthology.orchestra.run.smithi139.stdout:5.14.0-361.el9.x86_64 2024-04-15T21:19:24.305 INFO:teuthology.task.kernel:Running kernel on smithi139: 5.14.0-361.el9.x86_64 2024-04-15T21:19:24.305 DEBUG:teuthology.orchestra.run.smithi139:> sudo yum install -y kernel 2024-04-15T21:19:26.546 INFO:teuthology.orchestra.run.smithi139.stdout:CentOS-9 - AppStream 10 MB/s | 6.7 MB 00:00 2024-04-15T21:19:26.559 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS-9 - AppStream 11 MB/s | 6.7 MB 00:00 2024-04-15T21:19:27.947 INFO:teuthology.orchestra.run.smithi139.stdout:CentOS-9 - BaseOS 7.1 MB/s | 2.0 MB 00:00 2024-04-15T21:19:28.143 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS-9 - BaseOS 4.2 MB/s | 2.0 MB 00:00 2024-04-15T21:19:29.629 INFO:teuthology.orchestra.run.smithi139.stdout:CentOS Stream 9 - BaseOS 6.0 MB/s | 8.0 MB 00:01 2024-04-15T21:19:30.500 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - BaseOS 4.0 MB/s | 8.0 MB 00:02 2024-04-15T21:19:31.755 INFO:teuthology.orchestra.run.smithi139.stdout:CentOS Stream 9 - AppStream 18 MB/s | 19 MB 00:01 2024-04-15T21:19:32.493 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - AppStream 21 MB/s | 19 MB 00:00 2024-04-15T21:19:35.830 INFO:teuthology.orchestra.run.smithi139.stdout:CentOS Stream 9 - CRB 11 MB/s | 6.2 MB 00:00 2024-04-15T21:19:37.314 INFO:teuthology.orchestra.run.smithi139.stdout:CentOS Stream 9 - Extras packages 47 kB/s | 16 kB 00:00 2024-04-15T21:19:37.993 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - CRB 3.1 MB/s | 6.2 MB 00:01 2024-04-15T21:19:39.141 INFO:teuthology.orchestra.run.smithi139.stdout:Extra Packages for Enterprise Linux 12 MB/s | 21 MB 00:01 2024-04-15T21:19:39.578 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - Extras packages 37 kB/s | 16 kB 00:00 2024-04-15T21:19:40.829 INFO:teuthology.orchestra.run.smithi083.stdout:Extra Packages for Enterprise Linux 19 MB/s | 21 MB 00:01 2024-04-15T21:19:44.232 INFO:teuthology.orchestra.run.smithi139.stdout:lab-extras 15 kB/s | 1.7 kB 00:00 2024-04-15T21:19:45.829 INFO:teuthology.orchestra.run.smithi083.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-04-15T21:19:46.588 INFO:teuthology.orchestra.run.smithi139.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-15T21:19:46.589 INFO:teuthology.orchestra.run.smithi139.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-15T21:19:46.677 INFO:teuthology.orchestra.run.smithi139.stdout:Dependencies resolved. 2024-04-15T21:19:46.689 INFO:teuthology.orchestra.run.smithi139.stdout:================================================================================ 2024-04-15T21:19:46.689 INFO:teuthology.orchestra.run.smithi139.stdout: Package Arch Version Repository Size 2024-04-15T21:19:46.689 INFO:teuthology.orchestra.run.smithi139.stdout:================================================================================ 2024-04-15T21:19:46.689 INFO:teuthology.orchestra.run.smithi139.stdout:Installing: 2024-04-15T21:19:46.689 INFO:teuthology.orchestra.run.smithi139.stdout: kernel x86_64 5.14.0-438.el9 CentOS-BaseOS 277 k 2024-04-15T21:19:46.690 INFO:teuthology.orchestra.run.smithi139.stdout:Installing dependencies: 2024-04-15T21:19:46.690 INFO:teuthology.orchestra.run.smithi139.stdout: kernel-core x86_64 5.14.0-438.el9 CentOS-BaseOS 15 M 2024-04-15T21:19:46.690 INFO:teuthology.orchestra.run.smithi139.stdout: kernel-modules x86_64 5.14.0-438.el9 CentOS-BaseOS 34 M 2024-04-15T21:19:46.690 INFO:teuthology.orchestra.run.smithi139.stdout: kernel-modules-core x86_64 5.14.0-438.el9 CentOS-BaseOS 28 M 2024-04-15T21:19:46.690 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-15T21:19:46.690 INFO:teuthology.orchestra.run.smithi139.stdout:Transaction Summary 2024-04-15T21:19:46.690 INFO:teuthology.orchestra.run.smithi139.stdout:================================================================================ 2024-04-15T21:19:46.690 INFO:teuthology.orchestra.run.smithi139.stdout:Install 4 Packages 2024-04-15T21:19:46.690 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-15T21:19:46.690 INFO:teuthology.orchestra.run.smithi139.stdout:Total download size: 77 M 2024-04-15T21:19:46.690 INFO:teuthology.orchestra.run.smithi139.stdout:Installed size: 123 M 2024-04-15T21:19:46.690 INFO:teuthology.orchestra.run.smithi139.stdout:Downloading Packages: 2024-04-15T21:19:46.858 INFO:teuthology.orchestra.run.smithi139.stdout:(1/4): kernel-5.14.0-438.el9.x86_64.rpm 1.6 MB/s | 277 kB 00:00 2024-04-15T21:19:47.325 INFO:teuthology.orchestra.run.smithi139.stdout:(2/4): kernel-core-5.14.0-438.el9.x86_64.rpm 24 MB/s | 15 MB 00:00 2024-04-15T21:19:47.975 INFO:teuthology.orchestra.run.smithi139.stdout:(3/4): kernel-modules-5.14.0-438.el9.x86_64.rpm 26 MB/s | 34 MB 00:01 2024-04-15T21:19:48.203 INFO:teuthology.orchestra.run.smithi083.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-15T21:19:48.203 INFO:teuthology.orchestra.run.smithi083.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-15T21:19:48.284 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-04-15T21:19:48.294 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-04-15T21:19:48.294 INFO:teuthology.orchestra.run.smithi083.stdout: Package Arch Version Repository Size 2024-04-15T21:19:48.294 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-04-15T21:19:48.294 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2024-04-15T21:19:48.295 INFO:teuthology.orchestra.run.smithi083.stdout: kernel x86_64 5.14.0-438.el9 CentOS-BaseOS 277 k 2024-04-15T21:19:48.295 INFO:teuthology.orchestra.run.smithi083.stdout:Installing dependencies: 2024-04-15T21:19:48.295 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-core x86_64 5.14.0-438.el9 CentOS-BaseOS 15 M 2024-04-15T21:19:48.295 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules x86_64 5.14.0-438.el9 CentOS-BaseOS 34 M 2024-04-15T21:19:48.295 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules-core x86_64 5.14.0-438.el9 CentOS-BaseOS 28 M 2024-04-15T21:19:48.295 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:19:48.295 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2024-04-15T21:19:48.295 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-04-15T21:19:48.295 INFO:teuthology.orchestra.run.smithi083.stdout:Install 4 Packages 2024-04-15T21:19:48.295 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:19:48.296 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 77 M 2024-04-15T21:19:48.296 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 123 M 2024-04-15T21:19:48.296 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2024-04-15T21:19:48.439 INFO:teuthology.orchestra.run.smithi083.stdout:(1/4): kernel-5.14.0-438.el9.x86_64.rpm 1.9 MB/s | 277 kB 00:00 2024-04-15T21:19:48.586 INFO:teuthology.orchestra.run.smithi139.stdout:(4/4): kernel-modules-core-5.14.0-438.el9.x86_6 16 MB/s | 28 MB 00:01 2024-04-15T21:19:48.587 INFO:teuthology.orchestra.run.smithi139.stdout:-------------------------------------------------------------------------------- 2024-04-15T21:19:48.587 INFO:teuthology.orchestra.run.smithi139.stdout:Total 40 MB/s | 77 MB 00:01 2024-04-15T21:19:48.587 INFO:teuthology.orchestra.run.smithi139.stdout:Running transaction check 2024-04-15T21:19:48.933 INFO:teuthology.orchestra.run.smithi139.stdout:Transaction check succeeded. 2024-04-15T21:19:48.933 INFO:teuthology.orchestra.run.smithi139.stdout:Running transaction test 2024-04-15T21:19:49.255 INFO:teuthology.orchestra.run.smithi083.stdout:(2/4): kernel-core-5.14.0-438.el9.x86_64.rpm 16 MB/s | 15 MB 00:00 2024-04-15T21:19:49.790 INFO:teuthology.orchestra.run.smithi139.stdout:Transaction test succeeded. 2024-04-15T21:19:49.790 INFO:teuthology.orchestra.run.smithi139.stdout:Running transaction 2024-04-15T21:19:49.939 INFO:teuthology.orchestra.run.smithi083.stdout:(3/4): kernel-modules-5.14.0-438.el9.x86_64.rpm 20 MB/s | 34 MB 00:01 2024-04-15T21:19:50.231 INFO:teuthology.orchestra.run.smithi083.stdout:(4/4): kernel-modules-core-5.14.0-438.el9.x86_6 15 MB/s | 28 MB 00:01 2024-04-15T21:19:50.234 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2024-04-15T21:19:50.234 INFO:teuthology.orchestra.run.smithi083.stdout:Total 40 MB/s | 77 MB 00:01 2024-04-15T21:19:50.234 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2024-04-15T21:19:50.588 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2024-04-15T21:19:50.588 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2024-04-15T21:19:51.420 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2024-04-15T21:19:51.421 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2024-04-15T21:19:52.122 INFO:teuthology.orchestra.run.smithi139.stdout: Preparing : 1/1 2024-04-15T21:19:52.731 INFO:teuthology.orchestra.run.smithi139.stdout: Installing : kernel-modules-core-5.14.0-438.el9.x86_64 1/4 2024-04-15T21:19:52.816 INFO:teuthology.orchestra.run.smithi139.stdout: Installing : kernel-core-5.14.0-438.el9.x86_64 2/4 2024-04-15T21:19:53.707 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2024-04-15T21:19:54.342 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-modules-core-5.14.0-438.el9.x86_64 1/4 2024-04-15T21:19:54.457 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-core-5.14.0-438.el9.x86_64 2/4 2024-04-15T21:19:55.586 INFO:teuthology.orchestra.run.smithi139.stdout: Running scriptlet: kernel-core-5.14.0-438.el9.x86_64 2/4 2024-04-15T21:19:55.717 INFO:teuthology.orchestra.run.smithi139.stdout: Installing : kernel-modules-5.14.0-438.el9.x86_64 3/4 2024-04-15T21:19:57.201 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-core-5.14.0-438.el9.x86_64 2/4 2024-04-15T21:19:57.346 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-modules-5.14.0-438.el9.x86_64 3/4 2024-04-15T21:20:00.416 INFO:teuthology.orchestra.run.smithi139.stdout: Running scriptlet: kernel-modules-5.14.0-438.el9.x86_64 3/4 2024-04-15T21:20:00.467 INFO:teuthology.orchestra.run.smithi139.stdout: Installing : kernel-5.14.0-438.el9.x86_64 4/4 2024-04-15T21:20:02.043 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-modules-5.14.0-438.el9.x86_64 3/4 2024-04-15T21:20:02.153 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-5.14.0-438.el9.x86_64 4/4 2024-04-15T21:20:05.200 INFO:teuthology.orchestra.run.smithi139.stdout: Running scriptlet: kernel-modules-core-5.14.0-438.el9.x86_64 4/4 2024-04-15T21:20:06.845 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-modules-core-5.14.0-438.el9.x86_64 4/4 2024-04-15T21:20:34.363 INFO:teuthology.orchestra.run.smithi139.stdout: Running scriptlet: kernel-core-5.14.0-438.el9.x86_64 4/4 2024-04-15T21:20:34.363 INFO:teuthology.orchestra.run.smithi139.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-15T21:20:34.363 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-15T21:20:34.406 INFO:teuthology.orchestra.run.smithi139.stdout: Running scriptlet: kernel-modules-5.14.0-438.el9.x86_64 4/4 2024-04-15T21:20:34.716 INFO:teuthology.orchestra.run.smithi139.stdout: Running scriptlet: kernel-5.14.0-438.el9.x86_64 4/4 2024-04-15T21:20:34.716 INFO:teuthology.orchestra.run.smithi139.stdout: Verifying : kernel-5.14.0-438.el9.x86_64 1/4 2024-04-15T21:20:34.716 INFO:teuthology.orchestra.run.smithi139.stdout: Verifying : kernel-core-5.14.0-438.el9.x86_64 2/4 2024-04-15T21:20:34.716 INFO:teuthology.orchestra.run.smithi139.stdout: Verifying : kernel-modules-5.14.0-438.el9.x86_64 3/4 2024-04-15T21:20:35.125 INFO:teuthology.orchestra.run.smithi139.stdout: Verifying : kernel-modules-core-5.14.0-438.el9.x86_64 4/4 2024-04-15T21:20:35.125 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-15T21:20:35.125 INFO:teuthology.orchestra.run.smithi139.stdout:Installed: 2024-04-15T21:20:35.125 INFO:teuthology.orchestra.run.smithi139.stdout: kernel-5.14.0-438.el9.x86_64 2024-04-15T21:20:35.125 INFO:teuthology.orchestra.run.smithi139.stdout: kernel-core-5.14.0-438.el9.x86_64 2024-04-15T21:20:35.125 INFO:teuthology.orchestra.run.smithi139.stdout: kernel-modules-5.14.0-438.el9.x86_64 2024-04-15T21:20:35.126 INFO:teuthology.orchestra.run.smithi139.stdout: kernel-modules-core-5.14.0-438.el9.x86_64 2024-04-15T21:20:35.126 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-15T21:20:35.126 INFO:teuthology.orchestra.run.smithi139.stdout:Complete! 2024-04-15T21:20:35.457 DEBUG:teuthology.orchestra.run.smithi139:> rpm -q kernel | sort -rV | head -n 1 2024-04-15T21:20:35.553 INFO:teuthology.orchestra.run.smithi139.stdout:kernel-5.14.0-438.el9.x86_64 2024-04-15T21:20:35.554 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-438.el9.x86_64 2024-04-15T21:20:35.554 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-438.el9.x86_64 2024-04-15T21:20:35.554 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-15T21:20:35.554 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-15T21:20:35.554 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-04-15T21:20:35.554 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi139.front.sepia.ceph.com, path=None, version=distro) 2024-04-15T21:20:35.555 DEBUG:teuthology.orchestra.run.smithi139:> rpm -q kernel | sort -rV | head -n 1 2024-04-15T21:20:35.635 INFO:teuthology.orchestra.run.smithi139.stdout:kernel-5.14.0-438.el9.x86_64 2024-04-15T21:20:35.635 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-438.el9.x86_64 2024-04-15T21:20:35.635 DEBUG:teuthology.orchestra.run.smithi139:> sudo rpm -qi grub2-tools 2024-04-15T21:20:35.669 INFO:teuthology.orchestra.run.smithi139.stdout:Name : grub2-tools 2024-04-15T21:20:35.669 INFO:teuthology.orchestra.run.smithi139.stdout:Epoch : 1 2024-04-15T21:20:35.670 INFO:teuthology.orchestra.run.smithi139.stdout:Version : 2.06 2024-04-15T21:20:35.670 INFO:teuthology.orchestra.run.smithi139.stdout:Release : 68.el9 2024-04-15T21:20:35.670 INFO:teuthology.orchestra.run.smithi139.stdout:Architecture: x86_64 2024-04-15T21:20:35.670 INFO:teuthology.orchestra.run.smithi139.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-15T21:20:35.670 INFO:teuthology.orchestra.run.smithi139.stdout:Group : Unspecified 2024-04-15T21:20:35.670 INFO:teuthology.orchestra.run.smithi139.stdout:Size : 8264254 2024-04-15T21:20:35.670 INFO:teuthology.orchestra.run.smithi139.stdout:License : GPLv3+ 2024-04-15T21:20:35.670 INFO:teuthology.orchestra.run.smithi139.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-15T21:20:35.670 INFO:teuthology.orchestra.run.smithi139.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-15T21:20:35.670 INFO:teuthology.orchestra.run.smithi139.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-15T21:20:35.670 INFO:teuthology.orchestra.run.smithi139.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-15T21:20:35.670 INFO:teuthology.orchestra.run.smithi139.stdout:Packager : builder@centos.org 2024-04-15T21:20:35.670 INFO:teuthology.orchestra.run.smithi139.stdout:Vendor : CentOS 2024-04-15T21:20:35.671 INFO:teuthology.orchestra.run.smithi139.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-15T21:20:35.671 INFO:teuthology.orchestra.run.smithi139.stdout:Summary : Support tools for GRUB. 2024-04-15T21:20:35.671 INFO:teuthology.orchestra.run.smithi139.stdout:Description : 2024-04-15T21:20:35.671 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-15T21:20:35.671 INFO:teuthology.orchestra.run.smithi139.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-15T21:20:35.671 INFO:teuthology.orchestra.run.smithi139.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-15T21:20:35.671 INFO:teuthology.orchestra.run.smithi139.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-15T21:20:35.671 INFO:teuthology.orchestra.run.smithi139.stdout:hardware devices. 2024-04-15T21:20:35.671 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-15T21:20:35.671 INFO:teuthology.orchestra.run.smithi139.stdout:This subpackage provides tools for support of all platforms. 2024-04-15T21:20:35.672 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-15T21:20:35.672 INFO:teuthology.task.kernel:Updating grub on smithi139 to boot 5.14.0-438.el9.x86_64 2024-04-15T21:20:35.672 DEBUG:teuthology.orchestra.run.smithi139:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-15T21:20:36.100 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-core-5.14.0-438.el9.x86_64 4/4 2024-04-15T21:20:36.100 INFO:teuthology.orchestra.run.smithi083.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-15T21:20:36.100 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:20:36.144 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-modules-5.14.0-438.el9.x86_64 4/4 2024-04-15T21:20:36.277 INFO:teuthology.orchestra.run.smithi139.stderr:Generating grub configuration file ... 2024-04-15T21:20:36.529 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-5.14.0-438.el9.x86_64 4/4 2024-04-15T21:20:36.529 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-5.14.0-438.el9.x86_64 1/4 2024-04-15T21:20:36.530 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-core-5.14.0-438.el9.x86_64 2/4 2024-04-15T21:20:36.530 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-modules-5.14.0-438.el9.x86_64 3/4 2024-04-15T21:20:36.948 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-modules-core-5.14.0-438.el9.x86_64 4/4 2024-04-15T21:20:36.948 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:20:36.949 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2024-04-15T21:20:36.949 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-5.14.0-438.el9.x86_64 2024-04-15T21:20:36.949 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-core-5.14.0-438.el9.x86_64 2024-04-15T21:20:36.949 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules-5.14.0-438.el9.x86_64 2024-04-15T21:20:36.949 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules-core-5.14.0-438.el9.x86_64 2024-04-15T21:20:36.949 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:20:36.949 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-04-15T21:20:37.281 DEBUG:teuthology.orchestra.run.smithi083:> rpm -q kernel | sort -rV | head -n 1 2024-04-15T21:20:37.388 INFO:teuthology.orchestra.run.smithi083.stdout:kernel-5.14.0-438.el9.x86_64 2024-04-15T21:20:37.388 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-438.el9.x86_64 2024-04-15T21:20:37.388 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-438.el9.x86_64 2024-04-15T21:20:37.388 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-15T21:20:37.388 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-15T21:20:37.388 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-04-15T21:20:37.388 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi083.front.sepia.ceph.com, path=None, version=distro) 2024-04-15T21:20:37.388 DEBUG:teuthology.orchestra.run.smithi083:> rpm -q kernel | sort -rV | head -n 1 2024-04-15T21:20:37.468 INFO:teuthology.orchestra.run.smithi083.stdout:kernel-5.14.0-438.el9.x86_64 2024-04-15T21:20:37.469 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-438.el9.x86_64 2024-04-15T21:20:37.469 DEBUG:teuthology.orchestra.run.smithi083:> sudo rpm -qi grub2-tools 2024-04-15T21:20:37.513 INFO:teuthology.orchestra.run.smithi083.stdout:Name : grub2-tools 2024-04-15T21:20:37.513 INFO:teuthology.orchestra.run.smithi083.stdout:Epoch : 1 2024-04-15T21:20:37.513 INFO:teuthology.orchestra.run.smithi083.stdout:Version : 2.06 2024-04-15T21:20:37.513 INFO:teuthology.orchestra.run.smithi083.stdout:Release : 68.el9 2024-04-15T21:20:37.513 INFO:teuthology.orchestra.run.smithi083.stdout:Architecture: x86_64 2024-04-15T21:20:37.513 INFO:teuthology.orchestra.run.smithi083.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-15T21:20:37.513 INFO:teuthology.orchestra.run.smithi083.stdout:Group : Unspecified 2024-04-15T21:20:37.513 INFO:teuthology.orchestra.run.smithi083.stdout:Size : 8264254 2024-04-15T21:20:37.513 INFO:teuthology.orchestra.run.smithi083.stdout:License : GPLv3+ 2024-04-15T21:20:37.514 INFO:teuthology.orchestra.run.smithi083.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-15T21:20:37.514 INFO:teuthology.orchestra.run.smithi083.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-15T21:20:37.514 INFO:teuthology.orchestra.run.smithi083.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-15T21:20:37.514 INFO:teuthology.orchestra.run.smithi083.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-15T21:20:37.514 INFO:teuthology.orchestra.run.smithi083.stdout:Packager : builder@centos.org 2024-04-15T21:20:37.514 INFO:teuthology.orchestra.run.smithi083.stdout:Vendor : CentOS 2024-04-15T21:20:37.514 INFO:teuthology.orchestra.run.smithi083.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-15T21:20:37.514 INFO:teuthology.orchestra.run.smithi083.stdout:Summary : Support tools for GRUB. 2024-04-15T21:20:37.514 INFO:teuthology.orchestra.run.smithi083.stdout:Description : 2024-04-15T21:20:37.514 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:20:37.514 INFO:teuthology.orchestra.run.smithi083.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-15T21:20:37.514 INFO:teuthology.orchestra.run.smithi083.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-15T21:20:37.514 INFO:teuthology.orchestra.run.smithi083.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-15T21:20:37.514 INFO:teuthology.orchestra.run.smithi083.stdout:hardware devices. 2024-04-15T21:20:37.515 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:20:37.515 INFO:teuthology.orchestra.run.smithi083.stdout:This subpackage provides tools for support of all platforms. 2024-04-15T21:20:37.515 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-15T21:20:37.515 INFO:teuthology.task.kernel:Updating grub on smithi083 to boot 5.14.0-438.el9.x86_64 2024-04-15T21:20:37.516 DEBUG:teuthology.orchestra.run.smithi083:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-15T21:20:37.610 INFO:teuthology.orchestra.run.smithi139.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-15T21:20:37.648 INFO:teuthology.orchestra.run.smithi139.stderr:done 2024-04-15T21:20:37.652 DEBUG:teuthology.orchestra.run.smithi139:> mktemp 2024-04-15T21:20:37.673 INFO:teuthology.orchestra.run.smithi139.stdout:/tmp/tmp.Ww3iSXPjn4 2024-04-15T21:20:37.673 DEBUG:teuthology.orchestra.run.smithi139:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.Ww3iSXPjn4 2024-04-15T21:20:37.743 DEBUG:teuthology.orchestra.run.smithi139:> sudo chmod 0666 /tmp/tmp.Ww3iSXPjn4 2024-04-15T21:20:37.878 DEBUG:teuthology.orchestra.remote:smithi139:/tmp/tmp.Ww3iSXPjn4 is 6KB 2024-04-15T21:20:37.927 DEBUG:teuthology.orchestra.run.smithi139:> rm -fr /tmp/tmp.Ww3iSXPjn4 2024-04-15T21:20:37.940 DEBUG:teuthology.orchestra.run.smithi139:> sudo /bin/ls /boot/loader/entries || true 2024-04-15T21:20:38.006 INFO:teuthology.orchestra.run.smithi139.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-15T21:20:38.006 INFO:teuthology.orchestra.run.smithi139.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-15T21:20:38.006 INFO:teuthology.orchestra.run.smithi139.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-15T21:20:38.006 INFO:teuthology.orchestra.run.smithi139.stdout:15949e3c88704300882482eddfd7baae-5.14.0-438.el9.x86_64.conf 2024-04-15T21:20:38.007 DEBUG:teuthology.orchestra.run.smithi139:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-438.el9.x86_64 2024-04-15T21:20:38.142 INFO:teuthology.orchestra.run.smithi083.stderr:Generating grub configuration file ... 2024-04-15T21:20:38.155 DEBUG:teuthology.orchestra.run.smithi139:> sudo shutdown -r now 2024-04-15T21:20:39.569 INFO:teuthology.orchestra.run.smithi083.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-15T21:20:39.583 INFO:teuthology.orchestra.run.smithi083.stderr:done 2024-04-15T21:20:39.589 DEBUG:teuthology.orchestra.run.smithi083:> mktemp 2024-04-15T21:20:39.645 INFO:teuthology.orchestra.run.smithi083.stdout:/tmp/tmp.UbOwVlWIH3 2024-04-15T21:20:39.645 DEBUG:teuthology.orchestra.run.smithi083:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.UbOwVlWIH3 2024-04-15T21:20:39.713 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 0666 /tmp/tmp.UbOwVlWIH3 2024-04-15T21:20:39.855 DEBUG:teuthology.orchestra.remote:smithi083:/tmp/tmp.UbOwVlWIH3 is 6KB 2024-04-15T21:20:39.904 DEBUG:teuthology.orchestra.run.smithi083:> rm -fr /tmp/tmp.UbOwVlWIH3 2024-04-15T21:20:39.918 DEBUG:teuthology.orchestra.run.smithi083:> sudo /bin/ls /boot/loader/entries || true 2024-04-15T21:20:39.983 INFO:teuthology.orchestra.run.smithi083.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-15T21:20:39.983 INFO:teuthology.orchestra.run.smithi083.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-15T21:20:39.983 INFO:teuthology.orchestra.run.smithi083.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-15T21:20:39.984 INFO:teuthology.orchestra.run.smithi083.stdout:15949e3c88704300882482eddfd7baae-5.14.0-438.el9.x86_64.conf 2024-04-15T21:20:39.985 DEBUG:teuthology.orchestra.run.smithi083:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-438.el9.x86_64 2024-04-15T21:20:40.138 DEBUG:teuthology.orchestra.run.smithi083:> sudo shutdown -r now 2024-04-15T21:21:08.186 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-04-15T21:21:08.187 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi139.front.sepia.ceph.com' 2024-04-15T21:21:08.188 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi139.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-15T21:21:10.142 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-04-15T21:21:10.142 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-04-15T21:21:10.143 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-15T21:22:08.245 DEBUG:teuthology.orchestra.remote:timed out 2024-04-15T21:22:10.147 DEBUG:teuthology.orchestra.remote:timed out 2024-04-15T21:22:17.252 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi139.front.sepia.ceph.com' 2024-04-15T21:22:17.253 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi139.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-15T21:22:17.506 DEBUG:teuthology.orchestra.run.smithi139:> true 2024-04-15T21:22:18.171 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi139.front.sepia.ceph.com' 2024-04-15T21:22:18.171 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-438.el9.x86_64"... 2024-04-15T21:22:18.171 DEBUG:teuthology.orchestra.run.smithi139:> uname -r 2024-04-15T21:22:18.186 INFO:teuthology.orchestra.run.smithi139.stdout:5.14.0-438.el9.x86_64 2024-04-15T21:22:18.186 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-438.el9.x86_64 vs 5.14.0-438.el9.x86_64 2024-04-15T21:22:18.186 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-15T21:22:18.186 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-15T21:22:19.148 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-04-15T21:22:19.148 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-15T21:22:19.187 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-04-15T21:22:19.187 DEBUG:teuthology.orchestra.run.smithi139:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-15T21:22:19.264 INFO:teuthology.orchestra.run.smithi139.stdout:ttyS1 2024-04-15T21:22:19.316 DEBUG:teuthology.parallel:result is None 2024-04-15T21:22:22.206 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.83 2024-04-15T21:22:34.207 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-04-15T21:22:34.208 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-15T21:22:34.590 DEBUG:teuthology.orchestra.run.smithi083:> true 2024-04-15T21:22:34.933 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-04-15T21:22:34.933 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-438.el9.x86_64"... 2024-04-15T21:22:34.933 DEBUG:teuthology.orchestra.run.smithi083:> uname -r 2024-04-15T21:22:34.949 INFO:teuthology.orchestra.run.smithi083.stdout:5.14.0-438.el9.x86_64 2024-04-15T21:22:34.949 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-438.el9.x86_64 vs 5.14.0-438.el9.x86_64 2024-04-15T21:22:34.949 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-15T21:22:34.950 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-15T21:22:35.951 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-04-15T21:22:35.951 DEBUG:teuthology.orchestra.run.smithi083:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-15T21:22:36.021 INFO:teuthology.orchestra.run.smithi083.stdout:ttyS1 2024-04-15T21:22:36.061 DEBUG:teuthology.parallel:result is None 2024-04-15T21:22:36.061 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-15T21:22:36.068 INFO:teuthology.task.internal:Creating test directory... 2024-04-15T21:22:36.068 DEBUG:teuthology.orchestra.run.smithi083:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-15T21:22:36.071 DEBUG:teuthology.orchestra.run.smithi139:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-15T21:22:36.091 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-15T21:22:36.097 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-15T21:22:36.104 INFO:teuthology.task.internal:Creating archive directory... 2024-04-15T21:22:36.105 DEBUG:teuthology.orchestra.run.smithi083:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-15T21:22:36.129 DEBUG:teuthology.orchestra.run.smithi139:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-15T21:22:36.186 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-15T21:22:36.193 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-15T21:22:36.193 DEBUG:teuthology.orchestra.run.smithi083:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-15T21:22:36.196 DEBUG:teuthology.orchestra.run.smithi139:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-15T21:22:36.230 INFO:teuthology.orchestra.run.smithi083.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-15T21:22:36.232 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-15T21:22:36.244 INFO:teuthology.orchestra.run.smithi083.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-15T21:22:36.247 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-15T21:22:36.249 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-15T21:22:36.256 INFO:teuthology.task.internal:Configuring sudo... 2024-04-15T21:22:36.256 DEBUG:teuthology.orchestra.run.smithi083:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-15T21:22:36.289 DEBUG:teuthology.orchestra.run.smithi139:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-15T21:22:36.320 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-15T21:22:36.329 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-15T21:22:36.329 DEBUG:teuthology.orchestra.run.smithi083:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-15T21:22:36.359 DEBUG:teuthology.orchestra.run.smithi139:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-15T21:22:36.376 DEBUG:teuthology.orchestra.run.smithi083:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-15T21:22:36.454 DEBUG:teuthology.orchestra.run.smithi083:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-15T21:22:36.502 DEBUG:teuthology.orchestra.run.smithi083:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-15T21:22:36.585 DEBUG:teuthology.orchestra.run.smithi083:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-15T21:22:36.628 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-15T21:22:36.629 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-15T21:22:36.697 DEBUG:teuthology.orchestra.run.smithi139:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-15T21:22:36.743 DEBUG:teuthology.orchestra.run.smithi139:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-15T21:22:36.798 DEBUG:teuthology.orchestra.run.smithi139:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-15T21:22:36.881 DEBUG:teuthology.orchestra.run.smithi139:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-15T21:22:36.929 DEBUG:teuthology.orchestra.run.smithi139:> set -ex 2024-04-15T21:22:36.929 DEBUG:teuthology.orchestra.run.smithi139:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-15T21:22:37.000 DEBUG:teuthology.orchestra.run.smithi083:> sudo service rsyslog restart 2024-04-15T21:22:37.002 DEBUG:teuthology.orchestra.run.smithi139:> sudo service rsyslog restart 2024-04-15T21:22:37.046 INFO:teuthology.orchestra.run.smithi083.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-15T21:22:37.093 INFO:teuthology.orchestra.run.smithi139.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-15T21:22:37.603 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-15T21:22:37.610 INFO:teuthology.task.internal:Starting timer... 2024-04-15T21:22:37.610 INFO:teuthology.run_tasks:Running task pcp... 2024-04-15T21:22:37.622 INFO:teuthology.run_tasks:Running task selinux... 2024-04-15T21:22:37.631 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-04-15T21:22:37.632 DEBUG:teuthology.orchestra.run.smithi083:> sudo service auditd rotate 2024-04-15T21:22:37.696 INFO:teuthology.orchestra.run.smithi083.stdout:Rotating logs: 2024-04-15T21:22:37.698 DEBUG:teuthology.orchestra.run.smithi139:> sudo service auditd rotate 2024-04-15T21:22:37.774 INFO:teuthology.orchestra.run.smithi139.stdout:Rotating logs: 2024-04-15T21:22:37.779 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-15T21:22:37.779 DEBUG:teuthology.orchestra.run.smithi083:> /usr/sbin/getenforce 2024-04-15T21:22:37.804 INFO:teuthology.orchestra.run.smithi083.stdout:Permissive 2024-04-15T21:22:37.804 DEBUG:teuthology.orchestra.run.smithi139:> /usr/sbin/getenforce 2024-04-15T21:22:37.827 INFO:teuthology.orchestra.run.smithi139.stdout:Permissive 2024-04-15T21:22:37.827 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi083.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi139.front.sepia.ceph.com': 'permissive'} 2024-04-15T21:22:37.828 DEBUG:teuthology.orchestra.run.smithi083:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-15T21:22:37.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-15T21:22:37.873 DEBUG:teuthology.orchestra.run.smithi139:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-15T21:22:37.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-15T21:22:37.904 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-15T21:22:37.905 DEBUG:teuthology.orchestra.run.smithi083:> sudo /usr/sbin/setenforce permissive 2024-04-15T21:22:37.948 DEBUG:teuthology.orchestra.run.smithi139:> sudo /usr/sbin/setenforce permissive 2024-04-15T21:22:37.982 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-15T21:22:37.993 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-15T21:22:37.998 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-15T21:22:38.125 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-15T21:22:38.143 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-15T21:22:38.144 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi083.front.sepia.ceph.com,smithi139.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-15T21:26:58.930 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi083.front.sepia.ceph.com'), Remote(name='ubuntu@smithi139.front.sepia.ceph.com')] 2024-04-15T21:26:58.932 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-04-15T21:26:58.933 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-15T21:26:59.024 DEBUG:teuthology.orchestra.run.smithi083:> true 2024-04-15T21:26:59.101 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-04-15T21:26:59.102 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi139.front.sepia.ceph.com' 2024-04-15T21:26:59.102 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi139.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-15T21:26:59.189 DEBUG:teuthology.orchestra.run.smithi139:> true 2024-04-15T21:26:59.266 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi139.front.sepia.ceph.com' 2024-04-15T21:26:59.267 INFO:teuthology.run_tasks:Running task clock... 2024-04-15T21:26:59.277 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-15T21:26:59.277 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-15T21:26:59.277 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-15T21:26:59.280 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-15T21:26:59.280 DEBUG:teuthology.orchestra.run.smithi139:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-15T21:26:59.309 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-15T21:26:59.327 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-15T21:26:59.339 INFO:teuthology.orchestra.run.smithi139.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-15T21:26:59.357 INFO:teuthology.orchestra.run.smithi139.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-15T21:26:59.358 INFO:teuthology.orchestra.run.smithi083.stderr:sudo: ntpd: command not found 2024-04-15T21:26:59.372 INFO:teuthology.orchestra.run.smithi083.stdout:506 Cannot talk to daemon 2024-04-15T21:26:59.388 INFO:teuthology.orchestra.run.smithi139.stderr:sudo: ntpd: command not found 2024-04-15T21:26:59.390 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-15T21:26:59.402 INFO:teuthology.orchestra.run.smithi139.stdout:506 Cannot talk to daemon 2024-04-15T21:26:59.407 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-15T21:26:59.420 INFO:teuthology.orchestra.run.smithi139.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-15T21:26:59.438 INFO:teuthology.orchestra.run.smithi139.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-15T21:26:59.457 INFO:teuthology.orchestra.run.smithi083.stderr:bash: line 1: ntpq: command not found 2024-04-15T21:26:59.461 INFO:teuthology.orchestra.run.smithi083.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-15T21:26:59.461 INFO:teuthology.orchestra.run.smithi083.stdout:=============================================================================== 2024-04-15T21:26:59.461 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-15T21:26:59.461 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-15T21:26:59.461 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-15T21:26:59.461 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-15T21:26:59.491 INFO:teuthology.orchestra.run.smithi139.stderr:bash: line 1: ntpq: command not found 2024-04-15T21:26:59.494 INFO:teuthology.orchestra.run.smithi139.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-15T21:26:59.495 INFO:teuthology.orchestra.run.smithi139.stdout:=============================================================================== 2024-04-15T21:26:59.495 INFO:teuthology.orchestra.run.smithi139.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-15T21:26:59.495 INFO:teuthology.orchestra.run.smithi139.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-15T21:26:59.495 INFO:teuthology.orchestra.run.smithi139.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-15T21:26:59.495 INFO:teuthology.orchestra.run.smithi139.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-15T21:26:59.495 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-15T21:26:59.599 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'a9a752df26c63acad72e1b3569fd79a515ca0765'} 2024-04-15T21:26:59.599 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-04-15T21:26:59.600 INFO:tasks.cephadm:Cluster fsid is e4a42038-fb6e-11ee-bc8f-c7b262605968 2024-04-15T21:26:59.600 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-15T21:26:59.600 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-04-15T21:26:59.600 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi083': '172.21.15.83', 'mon.smithi139': '172.21.15.139'} 2024-04-15T21:26:59.600 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-15T21:26:59.600 DEBUG:teuthology.orchestra.run.smithi083:> sudo hostname $(hostname -s) 2024-04-15T21:26:59.630 DEBUG:teuthology.orchestra.run.smithi139:> sudo hostname $(hostname -s) 2024-04-15T21:26:59.659 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-04-15T21:26:59.660 DEBUG:teuthology.orchestra.run.smithi083:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-15T21:26:59.934 INFO:teuthology.orchestra.run.smithi083.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 15 21:26 /home/ubuntu/cephtest/cephadm 2024-04-15T21:26:59.934 DEBUG:teuthology.orchestra.run.smithi139:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-15T21:27:00.149 INFO:teuthology.orchestra.run.smithi139.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 15 21:27 /home/ubuntu/cephtest/cephadm 2024-04-15T21:27:00.150 DEBUG:teuthology.orchestra.run.smithi083:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-15T21:27:00.166 DEBUG:teuthology.orchestra.run.smithi139:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-15T21:27:00.192 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-04-15T21:27:00.192 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-15T21:27:00.209 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-15T21:27:01.071 INFO:teuthology.orchestra.run.smithi139.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-15T21:27:01.110 INFO:teuthology.orchestra.run.smithi083.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-15T21:27:25.692 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-04-15T21:27:25.692 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-15T21:27:25.692 INFO:teuthology.orchestra.run.smithi083.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-15T21:27:25.692 INFO:teuthology.orchestra.run.smithi083.stdout: "repo_digests": [ 2024-04-15T21:27:25.693 INFO:teuthology.orchestra.run.smithi083.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-15T21:27:25.693 INFO:teuthology.orchestra.run.smithi083.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-15T21:27:25.693 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-04-15T21:27:25.693 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-04-15T21:27:27.872 INFO:teuthology.orchestra.run.smithi139.stdout:{ 2024-04-15T21:27:27.872 INFO:teuthology.orchestra.run.smithi139.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-15T21:27:27.872 INFO:teuthology.orchestra.run.smithi139.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-15T21:27:27.872 INFO:teuthology.orchestra.run.smithi139.stdout: "repo_digests": [ 2024-04-15T21:27:27.872 INFO:teuthology.orchestra.run.smithi139.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-15T21:27:27.872 INFO:teuthology.orchestra.run.smithi139.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-15T21:27:27.872 INFO:teuthology.orchestra.run.smithi139.stdout: ] 2024-04-15T21:27:27.872 INFO:teuthology.orchestra.run.smithi139.stdout:} 2024-04-15T21:27:27.891 DEBUG:teuthology.orchestra.run.smithi083:> sudo mkdir -p /etc/ceph 2024-04-15T21:27:27.928 DEBUG:teuthology.orchestra.run.smithi139:> sudo mkdir -p /etc/ceph 2024-04-15T21:27:27.959 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 777 /etc/ceph 2024-04-15T21:27:28.001 DEBUG:teuthology.orchestra.run.smithi139:> sudo chmod 777 /etc/ceph 2024-04-15T21:27:28.029 INFO:tasks.cephadm:Writing seed config... 2024-04-15T21:27:28.030 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-15T21:27:28.030 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-15T21:27:28.031 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-15T21:27:28.031 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-15T21:27:28.031 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-15T21:27:28.031 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-15T21:27:28.031 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-15T21:27:28.031 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-15T21:27:28.031 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-15T21:27:28.032 DEBUG:teuthology.orchestra.run.smithi083:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-15T21:27:28.059 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = e4a42038-fb6e-11ee-bc8f-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-15T21:27:28.060 DEBUG:teuthology.orchestra.run.smithi083:mon.smithi083> sudo journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@mon.smithi083.service 2024-04-15T21:27:28.106 INFO:tasks.cephadm:Bootstrapping... 2024-04-15T21:27:28.106 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.83 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-15T21:27:28.306 INFO:teuthology.orchestra.run.smithi083.stderr:-------------------------------------------------------------------------------- 2024-04-15T21:27:28.307 INFO:teuthology.orchestra.run.smithi083.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'e4a42038-fb6e-11ee-bc8f-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.83', '--skip-admin-label'] 2024-04-15T21:27:28.330 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: 4.9.4-dev 2024-04-15T21:27:28.332 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying podman|docker is present... 2024-04-15T21:27:28.353 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: 4.9.4-dev 2024-04-15T21:27:28.357 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying lvm2 is present... 2024-04-15T21:27:28.357 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying time synchronization is in place... 2024-04-15T21:27:28.363 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-15T21:27:28.370 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-04-15T21:27:28.376 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: enabled 2024-04-15T21:27:28.382 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: active 2024-04-15T21:27:28.383 INFO:teuthology.orchestra.run.smithi083.stderr:Unit chronyd.service is enabled and running 2024-04-15T21:27:28.383 INFO:teuthology.orchestra.run.smithi083.stderr:Repeating the final host check... 2024-04-15T21:27:28.404 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: 4.9.4-dev 2024-04-15T21:27:28.406 INFO:teuthology.orchestra.run.smithi083.stderr:podman (/bin/podman) version 4.9.4 is present 2024-04-15T21:27:28.406 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl is present 2024-04-15T21:27:28.406 INFO:teuthology.orchestra.run.smithi083.stderr:lvcreate is present 2024-04-15T21:27:28.412 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-15T21:27:28.419 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-04-15T21:27:28.425 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: enabled 2024-04-15T21:27:28.432 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: active 2024-04-15T21:27:28.433 INFO:teuthology.orchestra.run.smithi083.stderr:Unit chronyd.service is enabled and running 2024-04-15T21:27:28.433 INFO:teuthology.orchestra.run.smithi083.stderr:Host looks OK 2024-04-15T21:27:28.433 INFO:teuthology.orchestra.run.smithi083.stderr:Cluster fsid: e4a42038-fb6e-11ee-bc8f-c7b262605968 2024-04-15T21:27:28.433 INFO:teuthology.orchestra.run.smithi083.stderr:Acquiring lock 139752356270432 on /run/cephadm/e4a42038-fb6e-11ee-bc8f-c7b262605968.lock 2024-04-15T21:27:28.433 INFO:teuthology.orchestra.run.smithi083.stderr:Lock 139752356270432 acquired on /run/cephadm/e4a42038-fb6e-11ee-bc8f-c7b262605968.lock 2024-04-15T21:27:28.433 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying IP 172.21.15.83 port 3300 ... 2024-04-15T21:27:28.434 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying IP 172.21.15.83 port 6789 ... 2024-04-15T21:27:28.434 INFO:teuthology.orchestra.run.smithi083.stderr:Base mon IP is 172.21.15.83, final addrv is [v2:172.21.15.83:3300,v1:172.21.15.83:6789] 2024-04-15T21:27:28.436 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.83 metric 100 2024-04-15T21:27:28.436 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.83 metric 100 2024-04-15T21:27:28.439 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-15T21:27:28.439 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-04-15T21:27:28.439 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1784sec hoplimit 64 pref medium 2024-04-15T21:27:28.441 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-15T21:27:28.442 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-15T21:27:28.442 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-15T21:27:28.442 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-15T21:27:28.442 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:ce91/64 scope link 2024-04-15T21:27:28.442 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-15T21:27:28.442 INFO:teuthology.orchestra.run.smithi083.stderr:Mon IP `172.21.15.83` is in CIDR network `172.21.0.0/20` 2024-04-15T21:27:28.442 INFO:teuthology.orchestra.run.smithi083.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-15T21:27:28.443 INFO:teuthology.orchestra.run.smithi083.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-15T21:27:28.543 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-04-15T21:27:29.101 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Getting image source signatures 2024-04-15T21:27:29.101 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-04-15T21:27:29.102 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-04-15T21:27:29.102 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-04-15T21:27:29.102 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-04-15T21:27:29.102 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-04-15T21:27:29.102 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-15T21:27:29.270 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Writing manifest to image destination 2024-04-15T21:27:29.422 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-15T21:27:29.830 INFO:teuthology.orchestra.run.smithi083.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-15T21:27:30.739 INFO:teuthology.orchestra.run.smithi083.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-15T21:27:30.739 INFO:teuthology.orchestra.run.smithi083.stderr:Extracting ceph user uid/gid from container image... 2024-04-15T21:27:31.138 INFO:teuthology.orchestra.run.smithi083.stderr:stat: 167 167 2024-04-15T21:27:31.754 INFO:teuthology.orchestra.run.smithi083.stderr:Creating initial keys... 2024-04-15T21:27:32.161 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-authtool: AQDEmx1mXo4BBRAAQht7367fR7nM6S0xb2La8A== 2024-04-15T21:27:33.133 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-authtool: AQDFmx1ma9vrAhAAShrDVNGm7ODjLaKw/veUjQ== 2024-04-15T21:27:34.047 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-authtool: AQDFmx1mqkZjOxAAkVR1XNZtOnDs5Ilnesorrg== 2024-04-15T21:27:34.530 INFO:teuthology.orchestra.run.smithi083.stderr:Creating initial monmap... 2024-04-15T21:27:34.895 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-15T21:27:34.895 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-04-15T21:27:34.895 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to e4a42038-fb6e-11ee-bc8f-c7b262605968 2024-04-15T21:27:34.896 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-15T21:27:35.526 INFO:teuthology.orchestra.run.smithi083.stderr:monmaptool for smithi083 [v2:172.21.15.83:3300,v1:172.21.15.83:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-15T21:27:35.526 INFO:teuthology.orchestra.run.smithi083.stderr:setting min_mon_release = octopus 2024-04-15T21:27:35.526 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: set fsid to e4a42038-fb6e-11ee-bc8f-c7b262605968 2024-04-15T21:27:35.526 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-15T21:27:35.526 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-15T21:27:35.527 INFO:teuthology.orchestra.run.smithi083.stderr:Creating mon... 2024-04-15T21:27:36.416 INFO:teuthology.orchestra.run.smithi083.stderr:create mon.smithi083 on 2024-04-15T21:27:36.632 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-15T21:27:36.775 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968.target → /etc/systemd/system/ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968.target. 2024-04-15T21:27:36.776 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968.target → /etc/systemd/system/ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968.target. 2024-04-15T21:27:37.087 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to reset failed state of unit ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@mon.smithi083.service: Unit ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@mon.smithi083.service not loaded. 2024-04-15T21:27:37.095 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968.target.wants/ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@mon.smithi083.service → /etc/systemd/system/ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@.service. 2024-04-15T21:27:37.756 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2024-04-15T21:27:37.763 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-04-15T21:27:37.764 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2024-04-15T21:27:37.764 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to enable service . firewalld.service is not available 2024-04-15T21:27:37.764 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mon to start... 2024-04-15T21:27:37.765 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mon... 2024-04-15T21:27:37.876 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:37 smithi083 systemd[1]: Started Ceph mon.smithi083 for e4a42038-fb6e-11ee-bc8f-c7b262605968. 2024-04-15T21:27:38.267 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: cluster: 2024-04-15T21:27:38.267 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: id: e4a42038-fb6e-11ee-bc8f-c7b262605968 2024-04-15T21:27:38.267 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-15T21:27:38.267 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-15T21:27:38.267 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: services: 2024-04-15T21:27:38.267 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi083 (age 0.528638s) 2024-04-15T21:27:38.267 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-15T21:27:38.267 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-15T21:27:38.267 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-15T21:27:38.267 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: data: 2024-04-15T21:27:38.267 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-15T21:27:38.267 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-15T21:27:38.267 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-15T21:27:38.267 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: pgs: 2024-04-15T21:27:38.268 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-15T21:27:38.842 INFO:teuthology.orchestra.run.smithi083.stderr:mon is available 2024-04-15T21:27:38.842 INFO:teuthology.orchestra.run.smithi083.stderr:Assimilating anything we can from ceph.conf... 2024-04-15T21:27:38.949 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:38 smithi083 ceph-mon[24448]: mon.smithi083 is new leader, mons smithi083 in quorum (ranks 0) 2024-04-15T21:27:38.949 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:38 smithi083 ceph-mon[24448]: monmap e1: 1 mons at {smithi083=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0]} 2024-04-15T21:27:38.949 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:38 smithi083 ceph-mon[24448]: fsmap 2024-04-15T21:27:38.949 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:38 smithi083 ceph-mon[24448]: osdmap e1: 0 total, 0 up, 0 in 2024-04-15T21:27:38.949 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:38 smithi083 ceph-mon[24448]: mgrmap e1: no daemons active 2024-04-15T21:27:38.949 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:38 smithi083 ceph-mon[24448]: from='client.? 172.21.15.83:0/3961659544' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-15T21:27:39.292 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-15T21:27:39.292 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [global] 2024-04-15T21:27:39.292 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: fsid = e4a42038-fb6e-11ee-bc8f-c7b262605968 2024-04-15T21:27:39.292 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.83:3300,v1:172.21.15.83:6789] 2024-04-15T21:27:39.292 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-15T21:27:39.293 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-15T21:27:39.293 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-15T21:27:39.293 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-15T21:27:39.293 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-15T21:27:39.293 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [mgr] 2024-04-15T21:27:39.293 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-15T21:27:39.293 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-15T21:27:39.293 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [osd] 2024-04-15T21:27:39.293 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-15T21:27:39.293 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-15T21:27:39.756 INFO:teuthology.orchestra.run.smithi083.stderr:Generating new minimal ceph.conf... 2024-04-15T21:27:39.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:39 smithi083 ceph-mon[24448]: from='client.? 172.21.15.83:0/2437949595' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-15T21:27:39.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:39 smithi083 ceph-mon[24448]: from='client.? 172.21.15.83:0/2437949595' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-15T21:27:40.794 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:40 smithi083 ceph-mon[24448]: from='client.? 172.21.15.83:0/1807814872' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:27:40.797 INFO:teuthology.orchestra.run.smithi083.stderr:Restarting the monitor... 2024-04-15T21:27:41.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:40 smithi083 systemd[1]: Stopping Ceph mon.smithi083 for e4a42038-fb6e-11ee-bc8f-c7b262605968... 2024-04-15T21:27:41.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:40 smithi083 bash[24667]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon.smithi083" found: no such container 2024-04-15T21:27:41.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:40 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi083[24442]: 2024-04-15T21:27:40.921+0000 7f89159e1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi083 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-15T21:27:41.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:40 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi083[24442]: 2024-04-15T21:27:40.921+0000 7f89159e1700 -1 mon.smithi083@0(leader) e1 *** Got Signal Terminated *** 2024-04-15T21:27:41.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:40 smithi083 podman[24678]: 2024-04-15 21:27:40.942128544 +0000 UTC m=+0.088727462 container died 90925438b89db26a39addd2177d0716967628fbb7c8d52458c9d3c1162f87a1b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi083, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, maintainer=Guillaume Abrioux , ceph=True, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, io.openshift.expose-services=, name=centos-stream, GIT_CLEAN=True, architecture=x86_64, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream) 2024-04-15T21:27:42.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:42 smithi083 podman[24678]: 2024-04-15 21:27:42.358863432 +0000 UTC m=+1.505462338 container cleanup 90925438b89db26a39addd2177d0716967628fbb7c8d52458c9d3c1162f87a1b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi083, architecture=x86_64, io.openshift.expose-services=, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, ceph=True, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream) 2024-04-15T21:27:42.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:42 smithi083 bash[24678]: ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi083 2024-04-15T21:27:42.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:42 smithi083 bash[24704]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon.smithi083" found: no such container 2024-04-15T21:27:44.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:44 smithi083 podman[24690]: 2024-04-15 21:27:44.266350494 +0000 UTC m=+3.319767756 container remove 90925438b89db26a39addd2177d0716967628fbb7c8d52458c9d3c1162f87a1b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi083, ceph=True, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, maintainer=Guillaume Abrioux , GIT_CLEAN=True, name=centos-stream, vcs-type=git, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, version=8, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, distribution-scope=public) 2024-04-15T21:27:44.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:44 smithi083 systemd[1]: ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@mon.smithi083.service: Deactivated successfully. 2024-04-15T21:27:44.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:44 smithi083 systemd[1]: Stopped Ceph mon.smithi083 for e4a42038-fb6e-11ee-bc8f-c7b262605968. 2024-04-15T21:27:44.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:44 smithi083 systemd[1]: Starting Ceph mon.smithi083 for e4a42038-fb6e-11ee-bc8f-c7b262605968... 2024-04-15T21:27:44.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:44 smithi083 podman[24771]: 2024-04-15 21:27:44.416440339 +0000 UTC m=+0.018343718 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-15T21:27:45.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:45 smithi083 podman[24771]: 2024-04-15 21:27:45.436906368 +0000 UTC m=+1.038809734 container create 4e1e2c99fe19bd8fa729652cdf80ec9ef25ddf983c1e094e9d33858a1bda80e0 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi083, distribution-scope=public, io.buildah.version=1.19.8, vcs-type=git, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, release=754, name=centos-stream, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, maintainer=Guillaume Abrioux ) 2024-04-15T21:27:47.725 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 podman[24771]: 2024-04-15 21:27:47.420451778 +0000 UTC m=+3.022355142 container init 4e1e2c99fe19bd8fa729652cdf80ec9ef25ddf983c1e094e9d33858a1bda80e0 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi083, vendor=Red Hat, Inc., distribution-scope=public, io.buildah.version=1.19.8, architecture=x86_64, ceph=True, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, vcs-type=git, maintainer=Guillaume Abrioux , name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8) 2024-04-15T21:27:47.725 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 podman[24771]: 2024-04-15 21:27:47.424969613 +0000 UTC m=+3.026872978 container start 4e1e2c99fe19bd8fa729652cdf80ec9ef25ddf983c1e094e9d33858a1bda80e0 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi083, distribution-scope=public, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, release=754, ceph=True, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, RELEASE=HEAD, com.redhat.component=centos-stream-container, name=centos-stream, vcs-type=git, io.openshift.expose-services=, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux ) 2024-04-15T21:27:47.725 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: set uid:gid to 167:167 (ceph:ceph) 2024-04-15T21:27:47.725 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-15T21:27:47.725 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: pidfile_write: ignore empty --pid-file 2024-04-15T21:27:47.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: load: jerasure load: lrc 2024-04-15T21:27:47.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: RocksDB version: 6.15.5 2024-04-15T21:27:47.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-15T21:27:47.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Compile date Apr 18 2022 2024-04-15T21:27:47.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: DB SUMMARY 2024-04-15T21:27:47.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: DB Session ID: D4T2PEIN2B7TCBOHP4O9 2024-04-15T21:27:47.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: CURRENT file: CURRENT 2024-04-15T21:27:47.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: IDENTITY file: IDENTITY 2024-04-15T21:27:47.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-15T21:27:47.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi083/store.db dir, Total Num: 1, files: 000008.sst 2024-04-15T21:27:47.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi083/store.db: 000010.log size: 82715 ; 2024-04-15T21:27:47.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.error_if_exists: 0 2024-04-15T21:27:47.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.create_if_missing: 0 2024-04-15T21:27:47.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.paranoid_checks: 1 2024-04-15T21:27:47.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-15T21:27:47.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.env: 0x5593bfcc6860 2024-04-15T21:27:47.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.fs: Posix File System 2024-04-15T21:27:47.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.info_log: 0x5593c0feffa0 2024-04-15T21:27:47.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_file_opening_threads: 16 2024-04-15T21:27:47.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.statistics: (nil) 2024-04-15T21:27:47.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.use_fsync: 0 2024-04-15T21:27:47.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_log_file_size: 0 2024-04-15T21:27:47.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-15T21:27:47.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-15T21:27:47.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.keep_log_file_num: 1000 2024-04-15T21:27:47.728 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.recycle_log_file_num: 0 2024-04-15T21:27:47.728 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.allow_fallocate: 1 2024-04-15T21:27:47.728 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.allow_mmap_reads: 0 2024-04-15T21:27:47.728 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.allow_mmap_writes: 0 2024-04-15T21:27:47.728 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.use_direct_reads: 0 2024-04-15T21:27:47.728 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-15T21:27:47.728 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.create_missing_column_families: 0 2024-04-15T21:27:47.728 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.db_log_dir: 2024-04-15T21:27:47.729 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi083/store.db 2024-04-15T21:27:47.729 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-15T21:27:47.729 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-15T21:27:47.729 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-15T21:27:47.729 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-15T21:27:47.729 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-15T21:27:47.729 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-15T21:27:47.729 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.advise_random_on_open: 1 2024-04-15T21:27:47.731 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.db_write_buffer_size: 0 2024-04-15T21:27:47.731 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.write_buffer_manager: 0x5593c10e0330 2024-04-15T21:27:47.731 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-15T21:27:47.731 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-15T21:27:47.731 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-15T21:27:47.731 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-15T21:27:47.731 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.rate_limiter: (nil) 2024-04-15T21:27:47.731 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-15T21:27:47.732 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.wal_recovery_mode: 2 2024-04-15T21:27:47.732 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.enable_thread_tracking: 0 2024-04-15T21:27:47.732 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.enable_pipelined_write: 0 2024-04-15T21:27:47.732 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.unordered_write: 0 2024-04-15T21:27:47.732 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-15T21:27:47.732 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-15T21:27:47.733 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-15T21:27:47.733 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-15T21:27:47.733 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.row_cache: None 2024-04-15T21:27:47.733 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.wal_filter: None 2024-04-15T21:27:47.733 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-15T21:27:47.733 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.allow_ingest_behind: 0 2024-04-15T21:27:47.733 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.preserve_deletes: 0 2024-04-15T21:27:47.734 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.two_write_queues: 0 2024-04-15T21:27:47.734 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.manual_wal_flush: 0 2024-04-15T21:27:47.734 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.atomic_flush: 0 2024-04-15T21:27:47.734 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-15T21:27:47.734 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-15T21:27:47.734 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-15T21:27:47.734 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.log_readahead_size: 0 2024-04-15T21:27:47.735 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-15T21:27:47.735 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.best_efforts_recovery: 0 2024-04-15T21:27:47.735 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-15T21:27:47.735 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-15T21:27:47.735 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.allow_data_in_errors: 0 2024-04-15T21:27:47.735 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.db_host_id: __hostname__ 2024-04-15T21:27:47.735 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_background_jobs: 2 2024-04-15T21:27:47.735 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_background_compactions: -1 2024-04-15T21:27:47.735 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_subcompactions: 1 2024-04-15T21:27:47.735 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-15T21:27:47.735 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-15T21:27:47.736 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-15T21:27:47.736 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_total_wal_size: 0 2024-04-15T21:27:47.736 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-15T21:27:47.736 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-15T21:27:47.736 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-15T21:27:47.736 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-15T21:27:47.736 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_open_files: -1 2024-04-15T21:27:47.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.bytes_per_sync: 0 2024-04-15T21:27:47.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-15T21:27:47.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-15T21:27:47.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compaction_readahead_size: 0 2024-04-15T21:27:47.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_background_flushes: -1 2024-04-15T21:27:47.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Compression algorithms supported: 2024-04-15T21:27:47.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-15T21:27:47.738 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: kZSTD supported: 0 2024-04-15T21:27:47.738 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: kXpressCompression supported: 0 2024-04-15T21:27:47.738 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: kLZ4HCCompression supported: 1 2024-04-15T21:27:47.738 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: kLZ4Compression supported: 1 2024-04-15T21:27:47.738 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: kBZip2Compression supported: 0 2024-04-15T21:27:47.738 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: kZlibCompression supported: 1 2024-04-15T21:27:47.738 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: kSnappyCompression supported: 1 2024-04-15T21:27:47.739 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-15T21:27:47.739 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi083/store.db/MANIFEST-000009 2024-04-15T21:27:47.739 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-15T21:27:47.739 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-15T21:27:47.739 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.merge_operator: 2024-04-15T21:27:47.739 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compaction_filter: None 2024-04-15T21:27:47.740 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compaction_filter_factory: None 2024-04-15T21:27:47.740 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.sst_partitioner_factory: None 2024-04-15T21:27:47.740 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-15T21:27:47.740 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-15T21:27:47.740 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5593c0fbbd00) 2024-04-15T21:27:47.740 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: cache_index_and_filter_blocks: 1 2024-04-15T21:27:47.741 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-15T21:27:47.741 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-15T21:27:47.741 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: pin_top_level_index_and_filter: 1 2024-04-15T21:27:47.741 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: index_type: 0 2024-04-15T21:27:47.741 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: data_block_index_type: 0 2024-04-15T21:27:47.741 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: index_shortening: 1 2024-04-15T21:27:47.742 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-15T21:27:47.742 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: hash_index_allow_collision: 1 2024-04-15T21:27:47.742 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: checksum: 1 2024-04-15T21:27:47.742 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: no_block_cache: 0 2024-04-15T21:27:47.742 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache: 0x5593c1026170 2024-04-15T21:27:47.742 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache_name: BinnedLRUCache 2024-04-15T21:27:47.742 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache_options: 2024-04-15T21:27:47.743 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: capacity : 536870912 2024-04-15T21:27:47.743 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: num_shard_bits : 4 2024-04-15T21:27:47.743 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: strict_capacity_limit : 0 2024-04-15T21:27:47.743 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: high_pri_pool_ratio: 0.000 2024-04-15T21:27:47.743 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache_compressed: (nil) 2024-04-15T21:27:47.743 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: persistent_cache: (nil) 2024-04-15T21:27:47.743 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_size: 4096 2024-04-15T21:27:47.744 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_size_deviation: 10 2024-04-15T21:27:47.744 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_restart_interval: 16 2024-04-15T21:27:47.744 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: index_block_restart_interval: 1 2024-04-15T21:27:47.744 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: metadata_block_size: 4096 2024-04-15T21:27:47.744 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: partition_filters: 0 2024-04-15T21:27:47.744 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: use_delta_encoding: 1 2024-04-15T21:27:47.745 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-15T21:27:47.745 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: whole_key_filtering: 1 2024-04-15T21:27:47.745 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: verify_compression: 0 2024-04-15T21:27:47.745 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: read_amp_bytes_per_bit: 0 2024-04-15T21:27:47.745 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: format_version: 4 2024-04-15T21:27:47.745 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: enable_index_compression: 1 2024-04-15T21:27:47.745 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_align: 0 2024-04-15T21:27:47.746 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.write_buffer_size: 33554432 2024-04-15T21:27:47.746 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_write_buffer_number: 2 2024-04-15T21:27:47.746 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compression: NoCompression 2024-04-15T21:27:47.746 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.bottommost_compression: Disabled 2024-04-15T21:27:47.746 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.prefix_extractor: nullptr 2024-04-15T21:27:47.746 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-15T21:27:47.747 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.num_levels: 7 2024-04-15T21:27:47.747 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-15T21:27:47.747 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-15T21:27:47.747 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-15T21:27:47.747 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-15T21:27:47.747 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-15T21:27:47.748 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-15T21:27:47.748 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-15T21:27:47.748 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-15T21:27:47.748 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-15T21:27:47.748 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-15T21:27:47.748 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-15T21:27:47.749 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compression_opts.level: 32767 2024-04-15T21:27:47.749 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compression_opts.strategy: 0 2024-04-15T21:27:47.749 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-15T21:27:47.749 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-15T21:27:47.749 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-15T21:27:47.749 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compression_opts.enabled: false 2024-04-15T21:27:47.749 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-15T21:27:47.750 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-15T21:27:47.750 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-15T21:27:47.750 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.target_file_size_base: 67108864 2024-04-15T21:27:47.750 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-15T21:27:47.750 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-15T21:27:47.750 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-15T21:27:47.751 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-15T21:27:47.751 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-15T21:27:47.751 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-15T21:27:47.751 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-15T21:27:47.751 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-15T21:27:47.751 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-15T21:27:47.751 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-15T21:27:47.752 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-15T21:27:47.752 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-15T21:27:47.752 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-15T21:27:47.752 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.arena_block_size: 4194304 2024-04-15T21:27:47.752 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-15T21:27:47.752 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-15T21:27:47.752 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-15T21:27:47.753 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.disable_auto_compactions: 0 2024-04-15T21:27:47.753 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-15T21:27:47.753 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-15T21:27:47.753 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-15T21:27:47.753 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-15T21:27:47.753 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-15T21:27:47.754 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-15T21:27:47.754 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-15T21:27:47.754 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-15T21:27:47.754 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-15T21:27:47.754 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-15T21:27:47.754 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.table_properties_collectors: 2024-04-15T21:27:47.754 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.inplace_update_support: 0 2024-04-15T21:27:47.754 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-15T21:27:47.754 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-15T21:27:47.755 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-15T21:27:47.755 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-15T21:27:47.755 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.bloom_locality: 0 2024-04-15T21:27:47.755 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.max_successive_merges: 0 2024-04-15T21:27:47.755 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-15T21:27:47.755 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.paranoid_file_checks: 0 2024-04-15T21:27:47.755 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.force_consistency_checks: 1 2024-04-15T21:27:47.755 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.report_bg_io_stats: 0 2024-04-15T21:27:47.755 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.ttl: 2592000 2024-04-15T21:27:47.756 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-15T21:27:47.756 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.enable_blob_files: false 2024-04-15T21:27:47.756 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.min_blob_size: 0 2024-04-15T21:27:47.756 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.blob_file_size: 268435456 2024-04-15T21:27:47.756 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-15T21:27:47.756 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-15T21:27:47.756 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-15T21:27:47.756 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi083/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-15T21:27:47.756 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-04-15T21:27:47.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-04-15T21:27:47.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713216467453785, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-15T21:27:47.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-04-15T21:27:47.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-15T21:27:47.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713216467454556, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79397, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77661, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71970, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1713216467, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "60d0b4d1-f729-40e6-8b74-30adcdc4ff1a", "db_session_id": "D4T2PEIN2B7TCBOHP4O9"}} 2024-04-15T21:27:47.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-04-15T21:27:47.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713216467454950, "job": 1, "event": "recovery_finished"} 2024-04-15T21:27:47.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi083/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-15T21:27:47.758 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x5593c100ca80 2024-04-15T21:27:47.758 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: DB pointer 0x5593c101c000 2024-04-15T21:27:47.758 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-15T21:27:47.758 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-15T21:27:47.758 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** DB Stats ** 2024-04-15T21:27:47.758 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-15T21:27:47.758 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-15T21:27:47.758 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-15T21:27:47.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-15T21:27:47.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-15T21:27:47.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-15T21:27:47.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-15T21:27:47.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-04-15T21:27:47.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** Compaction Stats [default] ** 2024-04-15T21:27:47.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-15T21:27:47.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-15T21:27:47.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 144.0 0.00 0.00 1 0.001 0 0 2024-04-15T21:27:47.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 144.0 0.00 0.00 1 0.001 0 0 2024-04-15T21:27:47.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 144.0 0.00 0.00 1 0.001 0 0 2024-04-15T21:27:47.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-04-15T21:27:47.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** Compaction Stats [default] ** 2024-04-15T21:27:47.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-15T21:27:47.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-15T21:27:47.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 144.0 0.00 0.00 1 0.001 0 0 2024-04-15T21:27:47.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-15T21:27:47.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-15T21:27:47.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-15T21:27:47.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-15T21:27:47.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-15T21:27:47.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-15T21:27:47.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative compaction: 0.00 GB write, 21.71 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-15T21:27:47.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval compaction: 0.00 GB write, 21.71 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-15T21:27:47.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-15T21:27:47.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-04-15T21:27:47.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-15T21:27:47.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-04-15T21:27:47.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** Compaction Stats [default] ** 2024-04-15T21:27:47.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-15T21:27:47.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-15T21:27:47.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 144.0 0.00 0.00 1 0.001 0 0 2024-04-15T21:27:47.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 144.0 0.00 0.00 1 0.001 0 0 2024-04-15T21:27:47.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-15T21:27:47.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-04-15T21:27:47.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** Compaction Stats [default] ** 2024-04-15T21:27:47.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-15T21:27:47.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-15T21:27:47.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 144.0 0.00 0.00 1 0.001 0 0 2024-04-15T21:27:47.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-15T21:27:47.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-15T21:27:47.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-15T21:27:47.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-15T21:27:47.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-15T21:27:47.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-15T21:27:47.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative compaction: 0.00 GB write, 21.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-15T21:27:47.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-15T21:27:47.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-15T21:27:47.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-04-15T21:27:47.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-15T21:27:47.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: starting mon.smithi083 rank 0 at public addrs [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] at bind addrs [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi083 fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 2024-04-15T21:27:47.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: mon.smithi083@-1(???) e1 preinit fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 2024-04-15T21:27:47.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: mon.smithi083@-1(???).mds e1 new map 2024-04-15T21:27:47.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: mon.smithi083@-1(???).mds e1 print_map 2024-04-15T21:27:47.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: e1 2024-04-15T21:27:47.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-15T21:27:47.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-15T21:27:47.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: legacy client fscid: -1 2024-04-15T21:27:47.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-04-15T21:27:47.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: No filesystems configured 2024-04-15T21:27:47.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: mon.smithi083@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-15T21:27:47.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: mon.smithi083@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-15T21:27:47.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: mon.smithi083@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-15T21:27:47.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: mon.smithi083@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-15T21:27:47.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: mon.smithi083@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-15T21:27:47.764 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: expand_channel_meta expand map: {default=false} 2024-04-15T21:27:47.764 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: expand_channel_meta from 'false' to 'false' 2024-04-15T21:27:47.764 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: expand_channel_meta expanded map: {default=false} 2024-04-15T21:27:47.764 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: expand_channel_meta expand map: {default=info} 2024-04-15T21:27:47.764 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: expand_channel_meta from 'info' to 'info' 2024-04-15T21:27:47.764 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: expand_channel_meta expanded map: {default=info} 2024-04-15T21:27:47.764 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: expand_channel_meta expand map: {default=daemon} 2024-04-15T21:27:47.764 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: expand_channel_meta from 'daemon' to 'daemon' 2024-04-15T21:27:47.765 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: expand_channel_meta expanded map: {default=daemon} 2024-04-15T21:27:47.765 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: expand_channel_meta expand map: {default=debug} 2024-04-15T21:27:47.765 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: expand_channel_meta from 'debug' to 'debug' 2024-04-15T21:27:47.765 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: expand_channel_meta expanded map: {default=debug} 2024-04-15T21:27:47.765 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: mon.smithi083 is new leader, mons smithi083 in quorum (ranks 0) 2024-04-15T21:27:47.765 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: monmap e1: 1 mons at {smithi083=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0]} 2024-04-15T21:27:47.765 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: fsmap 2024-04-15T21:27:47.766 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: osdmap e1: 0 total, 0 up, 0 in 2024-04-15T21:27:47.766 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:47 smithi083 ceph-mon[24790]: mgrmap e1: no daemons active 2024-04-15T21:27:48.449 INFO:teuthology.orchestra.run.smithi083.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-15T21:27:48.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:48 smithi083 bash[24771]: 4e1e2c99fe19bd8fa729652cdf80ec9ef25ddf983c1e094e9d33858a1bda80e0 2024-04-15T21:27:48.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:48 smithi083 systemd[1]: Started Ceph mon.smithi083 for e4a42038-fb6e-11ee-bc8f-c7b262605968. 2024-04-15T21:27:51.631 INFO:teuthology.orchestra.run.smithi083.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-15T21:27:51.631 INFO:teuthology.orchestra.run.smithi083.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-15T21:27:51.631 INFO:teuthology.orchestra.run.smithi083.stderr:Creating mgr... 2024-04-15T21:27:51.632 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying port 9283 ... 2024-04-15T21:27:51.798 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to reset failed state of unit ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@mgr.smithi083.kxqarn.service: Unit ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@mgr.smithi083.kxqarn.service not loaded. 2024-04-15T21:27:51.806 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968.target.wants/ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@mgr.smithi083.kxqarn.service → /etc/systemd/system/ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@.service. 2024-04-15T21:27:52.251 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:52 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/536708660' entity='client.admin' 2024-04-15T21:27:52.402 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2024-04-15T21:27:52.409 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-04-15T21:27:52.410 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2024-04-15T21:27:52.410 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to enable service . firewalld.service is not available 2024-04-15T21:27:52.418 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2024-04-15T21:27:52.425 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-04-15T21:27:52.426 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2024-04-15T21:27:52.426 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-15T21:27:52.426 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr to start... 2024-04-15T21:27:52.426 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr... 2024-04-15T21:27:52.931 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-15T21:27:52.931 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-04-15T21:27:52.931 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsid": "e4a42038-fb6e-11ee-bc8f-c7b262605968", 2024-04-15T21:27:52.931 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "health": { 2024-04-15T21:27:52.931 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-15T21:27:52.931 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "checks": {}, 2024-04-15T21:27:52.931 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mutes": [] 2024-04-15T21:27:52.931 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:27:52.931 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-15T21:27:52.932 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum": [ 2024-04-15T21:27:52.932 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 0 2024-04-15T21:27:52.932 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-04-15T21:27:52.932 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-15T21:27:52.932 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "smithi083" 2024-04-15T21:27:52.932 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-04-15T21:27:52.932 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-04-15T21:27:52.932 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "monmap": { 2024-04-15T21:27:52.932 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-15T21:27:52.932 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-15T21:27:52.932 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-15T21:27:52.932 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:27:52.932 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osdmap": { 2024-04-15T21:27:52.932 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-15T21:27:52.932 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-15T21:27:52.933 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-15T21:27:52.933 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-15T21:27:52.933 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-15T21:27:52.933 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-15T21:27:52.933 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-15T21:27:52.933 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:27:52.933 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgmap": { 2024-04-15T21:27:52.933 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-15T21:27:52.933 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-15T21:27:52.933 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-15T21:27:52.933 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-15T21:27:52.933 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-15T21:27:52.933 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-15T21:27:52.934 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-15T21:27:52.934 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-15T21:27:52.934 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:27:52.934 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsmap": { 2024-04-15T21:27:52.934 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-15T21:27:52.934 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-15T21:27:52.934 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-15T21:27:52.934 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:27:52.934 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-15T21:27:52.934 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": false, 2024-04-15T21:27:52.934 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-15T21:27:52.934 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modules": [ 2024-04-15T21:27:52.934 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "iostat", 2024-04-15T21:27:52.934 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "nfs", 2024-04-15T21:27:52.934 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "restful" 2024-04-15T21:27:52.935 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-04-15T21:27:52.935 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-04-15T21:27:52.935 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:27:52.935 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "servicemap": { 2024-04-15T21:27:52.935 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-15T21:27:52.935 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modified": "2024-04-15T21:27:37.736927+0000", 2024-04-15T21:27:52.935 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-04-15T21:27:52.935 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:27:52.935 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-15T21:27:52.935 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-04-15T21:27:53.570 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:53 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/3736465505' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-15T21:27:53.573 INFO:teuthology.orchestra.run.smithi083.stderr:mgr not available, waiting (1/15)... 2024-04-15T21:27:56.082 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-15T21:27:56.082 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-04-15T21:27:56.082 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsid": "e4a42038-fb6e-11ee-bc8f-c7b262605968", 2024-04-15T21:27:56.082 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "health": { 2024-04-15T21:27:56.082 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-15T21:27:56.082 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "checks": {}, 2024-04-15T21:27:56.082 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mutes": [] 2024-04-15T21:27:56.082 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:27:56.082 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-15T21:27:56.082 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum": [ 2024-04-15T21:27:56.083 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 0 2024-04-15T21:27:56.083 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-04-15T21:27:56.083 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-15T21:27:56.083 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "smithi083" 2024-04-15T21:27:56.083 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-04-15T21:27:56.083 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-04-15T21:27:56.083 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "monmap": { 2024-04-15T21:27:56.083 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-15T21:27:56.083 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-15T21:27:56.083 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-15T21:27:56.083 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:27:56.083 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osdmap": { 2024-04-15T21:27:56.083 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-15T21:27:56.083 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-15T21:27:56.083 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-15T21:27:56.084 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-15T21:27:56.084 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-15T21:27:56.084 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-15T21:27:56.084 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-15T21:27:56.084 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:27:56.084 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgmap": { 2024-04-15T21:27:56.084 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-15T21:27:56.084 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-15T21:27:56.084 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-15T21:27:56.084 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-15T21:27:56.084 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-15T21:27:56.084 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-15T21:27:56.084 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-15T21:27:56.084 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-15T21:27:56.085 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:27:56.085 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsmap": { 2024-04-15T21:27:56.085 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-15T21:27:56.085 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-15T21:27:56.085 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-15T21:27:56.085 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:27:56.085 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-15T21:27:56.085 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": false, 2024-04-15T21:27:56.085 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-15T21:27:56.085 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modules": [ 2024-04-15T21:27:56.085 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "iostat", 2024-04-15T21:27:56.085 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "nfs", 2024-04-15T21:27:56.085 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "restful" 2024-04-15T21:27:56.085 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-04-15T21:27:56.085 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-04-15T21:27:56.086 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:27:56.086 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "servicemap": { 2024-04-15T21:27:56.086 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-15T21:27:56.086 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modified": "2024-04-15T21:27:37.736927+0000", 2024-04-15T21:27:56.086 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-04-15T21:27:56.086 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:27:56.086 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-15T21:27:56.086 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-04-15T21:27:56.350 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:56 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/749290687' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-15T21:27:57.345 INFO:teuthology.orchestra.run.smithi083.stderr:mgr not available, waiting (2/15)... 2024-04-15T21:27:58.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:58 smithi083 ceph-mon[24790]: Activating manager daemon smithi083.kxqarn 2024-04-15T21:27:58.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:58 smithi083 ceph-mon[24790]: mgrmap e2: smithi083.kxqarn(active, starting, since 0.00273623s) 2024-04-15T21:27:58.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:58 smithi083 ceph-mon[24790]: from='mgr.14100 172.21.15.83:0/1617942549' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-15T21:27:58.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:58 smithi083 ceph-mon[24790]: from='mgr.14100 172.21.15.83:0/1617942549' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-15T21:27:58.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:58 smithi083 ceph-mon[24790]: from='mgr.14100 172.21.15.83:0/1617942549' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-15T21:27:58.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:58 smithi083 ceph-mon[24790]: from='mgr.14100 172.21.15.83:0/1617942549' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-04-15T21:27:58.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:58 smithi083 ceph-mon[24790]: from='mgr.14100 172.21.15.83:0/1617942549' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mgr metadata", "who": "smithi083.kxqarn", "id": "smithi083.kxqarn"}]: dispatch 2024-04-15T21:27:58.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:58 smithi083 ceph-mon[24790]: Manager daemon smithi083.kxqarn is now available 2024-04-15T21:27:58.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:58 smithi083 ceph-mon[24790]: from='mgr.14100 172.21.15.83:0/1617942549' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:27:58.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:58 smithi083 ceph-mon[24790]: from='mgr.14100 172.21.15.83:0/1617942549' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:27:58.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:58 smithi083 ceph-mon[24790]: from='mgr.14100 172.21.15.83:0/1617942549' entity='mgr.smithi083.kxqarn' 2024-04-15T21:27:58.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:58 smithi083 ceph-mon[24790]: from='mgr.14100 172.21.15.83:0/1617942549' entity='mgr.smithi083.kxqarn' 2024-04-15T21:27:58.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:27:58 smithi083 ceph-mon[24790]: from='mgr.14100 172.21.15.83:0/1617942549' entity='mgr.smithi083.kxqarn' 2024-04-15T21:27:59.995 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-15T21:27:59.995 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-04-15T21:27:59.995 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsid": "e4a42038-fb6e-11ee-bc8f-c7b262605968", 2024-04-15T21:27:59.995 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "health": { 2024-04-15T21:27:59.995 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-15T21:27:59.995 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "checks": {}, 2024-04-15T21:27:59.995 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mutes": [] 2024-04-15T21:27:59.995 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:27:59.995 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-15T21:27:59.995 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum": [ 2024-04-15T21:27:59.995 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 0 2024-04-15T21:27:59.996 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-04-15T21:27:59.996 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-15T21:27:59.996 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "smithi083" 2024-04-15T21:27:59.996 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-04-15T21:27:59.996 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-04-15T21:27:59.996 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "monmap": { 2024-04-15T21:27:59.996 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-15T21:27:59.996 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-15T21:27:59.996 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-15T21:27:59.996 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:27:59.996 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osdmap": { 2024-04-15T21:27:59.996 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-15T21:27:59.996 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-15T21:27:59.996 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-15T21:27:59.996 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-15T21:27:59.996 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-15T21:28:00.000 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-15T21:28:00.000 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-15T21:28:00.000 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:28:00.000 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgmap": { 2024-04-15T21:28:00.000 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-15T21:28:00.000 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-15T21:28:00.000 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-15T21:28:00.000 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-15T21:28:00.000 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-15T21:28:00.000 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-15T21:28:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-15T21:28:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-15T21:28:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:28:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsmap": { 2024-04-15T21:28:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-15T21:28:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-15T21:28:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-15T21:28:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:28:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-15T21:28:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": true, 2024-04-15T21:28:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-15T21:28:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modules": [ 2024-04-15T21:28:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "iostat", 2024-04-15T21:28:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "nfs", 2024-04-15T21:28:00.001 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "restful" 2024-04-15T21:28:00.002 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-04-15T21:28:00.002 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-04-15T21:28:00.002 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:28:00.002 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "servicemap": { 2024-04-15T21:28:00.002 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-15T21:28:00.002 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modified": "2024-04-15T21:27:37.736927+0000", 2024-04-15T21:28:00.002 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-04-15T21:28:00.002 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-15T21:28:00.002 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-15T21:28:00.002 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-04-15T21:28:00.350 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:00 smithi083 ceph-mon[24790]: mgrmap e3: smithi083.kxqarn(active, since 1.00533s) 2024-04-15T21:28:00.350 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:00 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/3019183027' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-15T21:28:00.551 INFO:teuthology.orchestra.run.smithi083.stderr:mgr is available 2024-04-15T21:28:01.091 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-15T21:28:01.091 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [global] 2024-04-15T21:28:01.091 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: fsid = e4a42038-fb6e-11ee-bc8f-c7b262605968 2024-04-15T21:28:01.092 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-15T21:28:01.092 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-15T21:28:01.092 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-15T21:28:01.092 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-15T21:28:01.092 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-15T21:28:01.092 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [mgr] 2024-04-15T21:28:01.092 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-15T21:28:01.092 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-15T21:28:01.092 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [osd] 2024-04-15T21:28:01.092 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-15T21:28:01.092 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-15T21:28:01.350 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:01 smithi083 ceph-mon[24790]: mgrmap e4: smithi083.kxqarn(active, since 2s) 2024-04-15T21:28:01.350 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:01 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/383925983' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-15T21:28:01.665 INFO:teuthology.orchestra.run.smithi083.stderr:Enabling cephadm module... 2024-04-15T21:28:02.652 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:02 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1825001768' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-15T21:28:03.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:03 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1825001768' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-15T21:28:03.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:03 smithi083 ceph-mon[24790]: mgrmap e5: smithi083.kxqarn(active, since 4s) 2024-04-15T21:28:03.673 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-04-15T21:28:03.673 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 5, 2024-04-15T21:28:03.673 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": true, 2024-04-15T21:28:03.673 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "active_name": "smithi083.kxqarn", 2024-04-15T21:28:03.673 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-15T21:28:03.674 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-04-15T21:28:04.170 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for the mgr to restart... 2024-04-15T21:28:04.170 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr epoch 5... 2024-04-15T21:28:04.397 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:04 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/3296968903' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-15T21:28:08.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:08 smithi083 ceph-mon[24790]: Active manager daemon smithi083.kxqarn restarted 2024-04-15T21:28:08.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:08 smithi083 ceph-mon[24790]: Activating manager daemon smithi083.kxqarn 2024-04-15T21:28:08.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:08 smithi083 ceph-mon[24790]: osdmap e2: 0 total, 0 up, 0 in 2024-04-15T21:28:09.480 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-04-15T21:28:09.480 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-04-15T21:28:09.480 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "initialized": true 2024-04-15T21:28:09.480 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-04-15T21:28:09.789 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:09 smithi083 ceph-mon[24790]: mgrmap e6: smithi083.kxqarn(active, starting, since 0.299942s) 2024-04-15T21:28:09.789 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:09 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-04-15T21:28:09.790 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:09 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mgr metadata", "who": "smithi083.kxqarn", "id": "smithi083.kxqarn"}]: dispatch 2024-04-15T21:28:09.790 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:09 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-15T21:28:09.790 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:09 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-15T21:28:09.790 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:09 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-15T21:28:09.790 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:09 smithi083 ceph-mon[24790]: Manager daemon smithi083.kxqarn is now available 2024-04-15T21:28:09.790 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:09 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:09.790 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:09 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:09.790 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:09 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:28:09.790 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:09 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:28:09.790 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:09 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:28:09.790 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:09 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:28:09.790 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:09 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:28:10.076 INFO:teuthology.orchestra.run.smithi083.stderr:mgr epoch 5 is available 2024-04-15T21:28:10.076 INFO:teuthology.orchestra.run.smithi083.stderr:Setting orchestrator backend to cephadm... 2024-04-15T21:28:10.751 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:10 smithi083 ceph-mon[24790]: mgrmap e7: smithi083.kxqarn(active, since 1.30246s) 2024-04-15T21:28:10.752 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:10 smithi083 ceph-mon[24790]: [15/Apr/2024:21:28:09] ENGINE Bus STARTING 2024-04-15T21:28:10.752 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:10 smithi083 ceph-mon[24790]: [15/Apr/2024:21:28:09] ENGINE Serving on https://172.21.15.83:7150 2024-04-15T21:28:10.752 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:10 smithi083 ceph-mon[24790]: [15/Apr/2024:21:28:09] ENGINE Bus STARTED 2024-04-15T21:28:10.752 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:10 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:10.752 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:10 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:28:11.878 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: value unchanged 2024-04-15T21:28:12.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:11 smithi083 ceph-mon[24790]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:28:12.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:11 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:12.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:11 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:28:12.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:11 smithi083 ceph-mon[24790]: mgrmap e8: smithi083.kxqarn(active, since 2s) 2024-04-15T21:28:12.464 INFO:teuthology.orchestra.run.smithi083.stderr:Generating ssh key... 2024-04-15T21:28:13.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:12 smithi083 ceph-mon[24790]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:28:14.424 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKtFqu265TbGP5bJmooe0LviVThx6J0RNX8e35WbXhJU7QBnJ+qEOiGUURlU9zTwTGXB5/Vgpvp3wXgQ86D4g7aDHvLXhtcYrh2IDjLvfp5OTtjfPVAIySpYpahOKK+8XgkxQ8tiX0RPJh3UcAjoHFMsCY3n9QojQl8JRfeW9VmfJrIZ2hMQJ9uZyitJHdbwyj3fSe11OG1iYDY26LSosSE+mBgBiE3oDOVo/J8EXWF4fAz6O0jzh6I0DfiBzzICxgZIk97u4DA4tzbLB+DteAUijoY5KywJh5xjCtdiWOwkDWT6CbkV7sWh30o+rK0rdN4COSmx9pqKO5KRYfdmpj2qkORWrspfR83EReAAQJQoTgd/wIkRzUpAbUtJzg2JgqruwQV82zs525pUlHY6R821aW+mC8MukzxZ6KzH54I3+Kmh9Co6Mh8BTJTOdXqbsAmnfJjv1vRA9WvcdN4bUqede605wsJQhjHJDqARy6+qAbrsbNngXWZR7HiJO8umM= ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968 2024-04-15T21:28:14.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:14 smithi083 ceph-mon[24790]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:28:14.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:14 smithi083 ceph-mon[24790]: Generating ssh key... 2024-04-15T21:28:14.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:14 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:14.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:14 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:15.010 INFO:teuthology.orchestra.run.smithi083.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-15T21:28:15.010 INFO:teuthology.orchestra.run.smithi083.stderr:Adding key to root@localhost authorized_keys... 2024-04-15T21:28:15.010 INFO:teuthology.orchestra.run.smithi083.stderr:Adding host smithi083... 2024-04-15T21:28:15.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:15 smithi083 ceph-mon[24790]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:28:16.639 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Added host 'smithi083' with addr '172.21.15.83' 2024-04-15T21:28:16.650 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:16 smithi083 ceph-mon[24790]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi083", "addr": "172.21.15.83", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:28:16.650 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:16 smithi083 ceph-mon[24790]: Deploying cephadm binary to smithi083 2024-04-15T21:28:17.259 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying mon service with default placement... 2024-04-15T21:28:17.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:17 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:17.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:17 smithi083 ceph-mon[24790]: Added host smithi083 2024-04-15T21:28:17.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:17 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:28:17.878 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-15T21:28:18.651 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying mgr service with default placement... 2024-04-15T21:28:19.069 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:18 smithi083 ceph-mon[24790]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:28:19.069 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:18 smithi083 ceph-mon[24790]: Saving service mon spec with placement count:5 2024-04-15T21:28:19.069 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:18 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:19.285 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-15T21:28:19.793 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying crash service with default placement... 2024-04-15T21:28:20.431 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled crash update... 2024-04-15T21:28:20.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:20 smithi083 ceph-mon[24790]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:28:20.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:20 smithi083 ceph-mon[24790]: Saving service mgr spec with placement count:2 2024-04-15T21:28:20.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:20 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:21.093 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying prometheus service with default placement... 2024-04-15T21:28:21.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:21 smithi083 ceph-mon[24790]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:28:21.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:21 smithi083 ceph-mon[24790]: Saving service crash spec with placement * 2024-04-15T21:28:21.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:21 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:22.318 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-04-15T21:28:22.876 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying grafana service with default placement... 2024-04-15T21:28:22.998 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:22 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:22.998 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:22 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:22.998 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:22 smithi083 ceph-mon[24790]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:28:22.998 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:22 smithi083 ceph-mon[24790]: Saving service prometheus spec with placement count:1 2024-04-15T21:28:22.998 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:22 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:23.514 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-04-15T21:28:24.167 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying node-exporter service with default placement... 2024-04-15T21:28:24.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:24 smithi083 ceph-mon[24790]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:28:24.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:24 smithi083 ceph-mon[24790]: Saving service grafana spec with placement count:1 2024-04-15T21:28:24.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:24 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:25.134 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-04-15T21:28:25.717 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying alertmanager service with default placement... 2024-04-15T21:28:26.009 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:25 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:26.010 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:25 smithi083 ceph-mon[24790]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:28:26.010 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:25 smithi083 ceph-mon[24790]: Saving service node-exporter spec with placement * 2024-04-15T21:28:26.010 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:25 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:26.010 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:25 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:26.311 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-04-15T21:28:27.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:27 smithi083 ceph-mon[24790]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:28:27.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:27 smithi083 ceph-mon[24790]: Saving service alertmanager spec with placement count:1 2024-04-15T21:28:27.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:27 smithi083 ceph-mon[24790]: from='mgr.14118 172.21.15.83:0/2659982681' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:28.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:28 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1543873949' entity='client.admin' 2024-04-15T21:28:29.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:29 smithi083 ceph-mon[24790]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:28:29.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:29 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1153531916' entity='client.admin' 2024-04-15T21:28:30.185 INFO:teuthology.orchestra.run.smithi083.stderr:Enabling the dashboard module... 2024-04-15T21:28:32.006 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:31 smithi083 ceph-mon[24790]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:28:32.006 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:31 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/3091664284' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-15T21:28:32.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:32 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/3091664284' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-15T21:28:32.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:32 smithi083 ceph-mon[24790]: mgrmap e9: smithi083.kxqarn(active, since 23s) 2024-04-15T21:28:33.033 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-04-15T21:28:33.033 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 9, 2024-04-15T21:28:33.033 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": true, 2024-04-15T21:28:33.033 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "active_name": "smithi083.kxqarn", 2024-04-15T21:28:33.034 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-15T21:28:33.034 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-04-15T21:28:33.683 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for the mgr to restart... 2024-04-15T21:28:33.683 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr epoch 9... 2024-04-15T21:28:33.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:33 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1019148001' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-15T21:28:38.720 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:38 smithi083 ceph-mon[24790]: Active manager daemon smithi083.kxqarn restarted 2024-04-15T21:28:38.720 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:38 smithi083 ceph-mon[24790]: Activating manager daemon smithi083.kxqarn 2024-04-15T21:28:38.720 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:38 smithi083 ceph-mon[24790]: osdmap e3: 0 total, 0 up, 0 in 2024-04-15T21:28:39.484 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-04-15T21:28:39.484 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-04-15T21:28:39.484 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "initialized": true 2024-04-15T21:28:39.485 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-04-15T21:28:39.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:39 smithi083 ceph-mon[24790]: mgrmap e10: smithi083.kxqarn(active, starting, since 0.904877s) 2024-04-15T21:28:39.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:39 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-04-15T21:28:39.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:39 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mgr metadata", "who": "smithi083.kxqarn", "id": "smithi083.kxqarn"}]: dispatch 2024-04-15T21:28:39.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:39 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-15T21:28:39.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:39 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-15T21:28:39.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:39 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-15T21:28:39.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:39 smithi083 ceph-mon[24790]: Manager daemon smithi083.kxqarn is now available 2024-04-15T21:28:39.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:39 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:28:39.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:39 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:28:39.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:39 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:28:39.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:39 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:28:39.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:39 smithi083 ceph-mon[24790]: [15/Apr/2024:21:28:38] ENGINE Bus STARTING 2024-04-15T21:28:39.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:39 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:40.326 INFO:teuthology.orchestra.run.smithi083.stderr:mgr epoch 9 is available 2024-04-15T21:28:40.326 INFO:teuthology.orchestra.run.smithi083.stderr:Generating a dashboard self-signed certificate... 2024-04-15T21:28:40.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:40 smithi083 ceph-mon[24790]: [15/Apr/2024:21:28:38] ENGINE Serving on https://172.21.15.83:7150 2024-04-15T21:28:40.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:40 smithi083 ceph-mon[24790]: [15/Apr/2024:21:28:38] ENGINE Bus STARTED 2024-04-15T21:28:40.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:40 smithi083 ceph-mon[24790]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-15T21:28:40.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:40 smithi083 ceph-mon[24790]: mgrmap e11: smithi083.kxqarn(active, since 1.90701s) 2024-04-15T21:28:40.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:40 smithi083 ceph-mon[24790]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-15T21:28:40.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:40 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:41.010 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-15T21:28:41.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:41 smithi083 ceph-mon[24790]: mgrmap e12: smithi083.kxqarn(active, since 2s) 2024-04-15T21:28:41.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:41 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:41.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:41 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:41.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:41 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:41.649 INFO:teuthology.orchestra.run.smithi083.stderr:Creating initial admin user... 2024-04-15T21:28:42.510 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$C5qJvCfCCffmNQtZJwyP5OWXoQz/2uzEwmi/JHRcx7I0vaeszQ9vm", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1713216522, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-15T21:28:42.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:42 smithi083 ceph-mon[24790]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:28:43.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:43 smithi083 ceph-mon[24790]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:28:43.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:43 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:43.796 INFO:teuthology.orchestra.run.smithi083.stderr:Fetching dashboard port number... 2024-04-15T21:28:44.417 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 8443 2024-04-15T21:28:44.792 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:44 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/2168885583' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-15T21:28:45.113 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2024-04-15T21:28:45.120 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-04-15T21:28:45.121 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2024-04-15T21:28:45.121 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-15T21:28:45.123 INFO:teuthology.orchestra.run.smithi083.stderr:Ceph Dashboard is now available at: 2024-04-15T21:28:45.123 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-15T21:28:45.123 INFO:teuthology.orchestra.run.smithi083.stderr: URL: https://smithi083.front.sepia.ceph.com:8443/ 2024-04-15T21:28:45.123 INFO:teuthology.orchestra.run.smithi083.stderr: User: admin 2024-04-15T21:28:45.123 INFO:teuthology.orchestra.run.smithi083.stderr: Password: dvuj5j865y 2024-04-15T21:28:45.123 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-15T21:28:45.123 INFO:teuthology.orchestra.run.smithi083.stderr:Enabling autotune for osd_memory_target 2024-04-15T21:28:45.828 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:45 smithi083 ceph-mon[24790]: mgrmap e13: smithi083.kxqarn(active, since 6s) 2024-04-15T21:28:47.193 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-04-15T21:28:47.802 INFO:teuthology.orchestra.run.smithi083.stderr:You can access the Ceph CLI with: 2024-04-15T21:28:47.803 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-15T21:28:47.803 INFO:teuthology.orchestra.run.smithi083.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-15T21:28:47.803 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-15T21:28:47.803 INFO:teuthology.orchestra.run.smithi083.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-15T21:28:47.803 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-15T21:28:47.803 INFO:teuthology.orchestra.run.smithi083.stderr: ceph telemetry on 2024-04-15T21:28:47.803 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-15T21:28:47.803 INFO:teuthology.orchestra.run.smithi083.stderr:For more information see: 2024-04-15T21:28:47.803 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-15T21:28:47.803 INFO:teuthology.orchestra.run.smithi083.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-15T21:28:47.803 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-15T21:28:47.804 INFO:teuthology.orchestra.run.smithi083.stderr:Bootstrap complete. 2024-04-15T21:28:47.845 INFO:tasks.cephadm:Fetching config... 2024-04-15T21:28:47.845 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-15T21:28:47.845 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-15T21:28:47.862 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-15T21:28:47.862 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-15T21:28:47.862 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-15T21:28:47.917 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-15T21:28:47.917 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-15T21:28:47.917 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/keyring of=/dev/stdout 2024-04-15T21:28:47.987 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-15T21:28:48.348 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-15T21:28:48.348 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-15T21:28:48.349 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:48 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1594801102' entity='client.admin' 2024-04-15T21:28:48.365 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-15T21:28:48.366 DEBUG:teuthology.orchestra.run.smithi083:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKtFqu265TbGP5bJmooe0LviVThx6J0RNX8e35WbXhJU7QBnJ+qEOiGUURlU9zTwTGXB5/Vgpvp3wXgQ86D4g7aDHvLXhtcYrh2IDjLvfp5OTtjfPVAIySpYpahOKK+8XgkxQ8tiX0RPJh3UcAjoHFMsCY3n9QojQl8JRfeW9VmfJrIZ2hMQJ9uZyitJHdbwyj3fSe11OG1iYDY26LSosSE+mBgBiE3oDOVo/J8EXWF4fAz6O0jzh6I0DfiBzzICxgZIk97u4DA4tzbLB+DteAUijoY5KywJh5xjCtdiWOwkDWT6CbkV7sWh30o+rK0rdN4COSmx9pqKO5KRYfdmpj2qkORWrspfR83EReAAQJQoTgd/wIkRzUpAbUtJzg2JgqruwQV82zs525pUlHY6R821aW+mC8MukzxZ6KzH54I3+Kmh9Co6Mh8BTJTOdXqbsAmnfJjv1vRA9WvcdN4bUqede605wsJQhjHJDqARy6+qAbrsbNngXWZR7HiJO8umM= ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-15T21:28:48.448 INFO:teuthology.orchestra.run.smithi083.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKtFqu265TbGP5bJmooe0LviVThx6J0RNX8e35WbXhJU7QBnJ+qEOiGUURlU9zTwTGXB5/Vgpvp3wXgQ86D4g7aDHvLXhtcYrh2IDjLvfp5OTtjfPVAIySpYpahOKK+8XgkxQ8tiX0RPJh3UcAjoHFMsCY3n9QojQl8JRfeW9VmfJrIZ2hMQJ9uZyitJHdbwyj3fSe11OG1iYDY26LSosSE+mBgBiE3oDOVo/J8EXWF4fAz6O0jzh6I0DfiBzzICxgZIk97u4DA4tzbLB+DteAUijoY5KywJh5xjCtdiWOwkDWT6CbkV7sWh30o+rK0rdN4COSmx9pqKO5KRYfdmpj2qkORWrspfR83EReAAQJQoTgd/wIkRzUpAbUtJzg2JgqruwQV82zs525pUlHY6R821aW+mC8MukzxZ6KzH54I3+Kmh9Co6Mh8BTJTOdXqbsAmnfJjv1vRA9WvcdN4bUqede605wsJQhjHJDqARy6+qAbrsbNngXWZR7HiJO8umM= ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968 2024-04-15T21:28:48.463 DEBUG:teuthology.orchestra.run.smithi139:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKtFqu265TbGP5bJmooe0LviVThx6J0RNX8e35WbXhJU7QBnJ+qEOiGUURlU9zTwTGXB5/Vgpvp3wXgQ86D4g7aDHvLXhtcYrh2IDjLvfp5OTtjfPVAIySpYpahOKK+8XgkxQ8tiX0RPJh3UcAjoHFMsCY3n9QojQl8JRfeW9VmfJrIZ2hMQJ9uZyitJHdbwyj3fSe11OG1iYDY26LSosSE+mBgBiE3oDOVo/J8EXWF4fAz6O0jzh6I0DfiBzzICxgZIk97u4DA4tzbLB+DteAUijoY5KywJh5xjCtdiWOwkDWT6CbkV7sWh30o+rK0rdN4COSmx9pqKO5KRYfdmpj2qkORWrspfR83EReAAQJQoTgd/wIkRzUpAbUtJzg2JgqruwQV82zs525pUlHY6R821aW+mC8MukzxZ6KzH54I3+Kmh9Co6Mh8BTJTOdXqbsAmnfJjv1vRA9WvcdN4bUqede605wsJQhjHJDqARy6+qAbrsbNngXWZR7HiJO8umM= ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-15T21:28:48.513 INFO:teuthology.orchestra.run.smithi139.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDKtFqu265TbGP5bJmooe0LviVThx6J0RNX8e35WbXhJU7QBnJ+qEOiGUURlU9zTwTGXB5/Vgpvp3wXgQ86D4g7aDHvLXhtcYrh2IDjLvfp5OTtjfPVAIySpYpahOKK+8XgkxQ8tiX0RPJh3UcAjoHFMsCY3n9QojQl8JRfeW9VmfJrIZ2hMQJ9uZyitJHdbwyj3fSe11OG1iYDY26LSosSE+mBgBiE3oDOVo/J8EXWF4fAz6O0jzh6I0DfiBzzICxgZIk97u4DA4tzbLB+DteAUijoY5KywJh5xjCtdiWOwkDWT6CbkV7sWh30o+rK0rdN4COSmx9pqKO5KRYfdmpj2qkORWrspfR83EReAAQJQoTgd/wIkRzUpAbUtJzg2JgqruwQV82zs525pUlHY6R821aW+mC8MukzxZ6KzH54I3+Kmh9Co6Mh8BTJTOdXqbsAmnfJjv1vRA9WvcdN4bUqede605wsJQhjHJDqARy6+qAbrsbNngXWZR7HiJO8umM= ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968 2024-04-15T21:28:48.529 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-15T21:28:51.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:51 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:51.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:51 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd/host:smithi083", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:28:51.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:51 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:51.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:51 smithi083 ceph-mon[24790]: Deploying daemon alertmanager.smithi083 on smithi083 2024-04-15T21:28:52.803 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:52 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/2111861096' entity='client.admin' 2024-04-15T21:28:52.828 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-15T21:28:52.828 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-15T21:28:56.350 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:56 smithi083 ceph-mon[24790]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:28:56.350 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:56 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:28:56.559 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi139 2024-04-15T21:28:56.559 DEBUG:teuthology.orchestra.run.smithi139:> set -ex 2024-04-15T21:28:56.559 DEBUG:teuthology.orchestra.run.smithi139:> dd of=/etc/ceph/ceph.conf 2024-04-15T21:28:56.578 DEBUG:teuthology.orchestra.run.smithi139:> set -ex 2024-04-15T21:28:56.579 DEBUG:teuthology.orchestra.run.smithi139:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-15T21:28:56.637 INFO:tasks.cephadm:Adding host smithi139 to orchestrator... 2024-04-15T21:28:56.637 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph orch host add smithi139 2024-04-15T21:28:59.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:28:59 smithi083 ceph-mon[24790]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:01.800 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:01 smithi083 ceph-mon[24790]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:02.388 INFO:teuthology.orchestra.run.smithi083.stdout:Added host 'smithi139' with addr '172.21.15.139' 2024-04-15T21:29:02.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:02 smithi083 ceph-mon[24790]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi139", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:29:02.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:02 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:02.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:02 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-15T21:29:02.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:02 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-15T21:29:02.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:02 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:29:02.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:02 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:03.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:03 smithi083 ceph-mon[24790]: Deploying daemon crash.smithi083 on smithi083 2024-04-15T21:29:03.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:03 smithi083 ceph-mon[24790]: Deploying cephadm binary to smithi139 2024-04-15T21:29:03.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:03 smithi083 ceph-mon[24790]: Added host smithi139 2024-04-15T21:29:03.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:03 smithi083 ceph-mon[24790]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:03.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:03 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:03.900 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph orch host ls --format=json 2024-04-15T21:29:05.836 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:05 smithi083 ceph-mon[24790]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:05.836 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:05 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:05.836 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:05 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:05.836 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:05 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:05.836 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:05 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-15T21:29:05.836 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:05 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:06.230 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:29:06.230 INFO:teuthology.orchestra.run.smithi083.stdout:[{"addr": "172.21.15.83", "hostname": "smithi083", "labels": [], "status": ""}, {"addr": "172.21.15.139", "hostname": "smithi139", "labels": [], "status": ""}] 2024-04-15T21:29:06.919 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:06 smithi083 ceph-mon[24790]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-15T21:29:06.919 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:06 smithi083 ceph-mon[24790]: Deploying daemon grafana.smithi083 on smithi083 2024-04-15T21:29:06.941 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-15T21:29:06.941 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph osd crush tunables default 2024-04-15T21:29:07.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:07 smithi083 ceph-mon[24790]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-15T21:29:07.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:07 smithi083 ceph-mon[24790]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:09.742 INFO:teuthology.orchestra.run.smithi083.stderr:adjusted tunables profile to default 2024-04-15T21:29:09.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:09 smithi083 ceph-mon[24790]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:09.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:09 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:09.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:09 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/588203402' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-15T21:29:10.440 INFO:tasks.cephadm:Adding mon.smithi083 on smithi083 2024-04-15T21:29:10.440 INFO:tasks.cephadm:Adding mon.smithi139 on smithi139 2024-04-15T21:29:10.440 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph orch apply mon '2;smithi083:172.21.15.83=smithi083;smithi139:172.21.15.139=smithi139' 2024-04-15T21:29:11.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:10 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/588203402' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-15T21:29:11.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:10 smithi083 ceph-mon[24790]: osdmap e4: 0 total, 0 up, 0 in 2024-04-15T21:29:11.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:11 smithi083 ceph-mon[24790]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:12.198 INFO:teuthology.orchestra.run.smithi139.stdout:Scheduled mon update... 2024-04-15T21:29:12.733 DEBUG:teuthology.orchestra.run.smithi139:mon.smithi139> sudo journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@mon.smithi139.service 2024-04-15T21:29:12.738 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-15T21:29:12.739 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph mon dump -f json 2024-04-15T21:29:13.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:13 smithi083 ceph-mon[24790]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi083:172.21.15.83=smithi083;smithi139:172.21.15.139=smithi139", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:29:13.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:13 smithi083 ceph-mon[24790]: Saving service mon spec with placement smithi083:172.21.15.83=smithi083;smithi139:172.21.15.139=smithi139;count:2 2024-04-15T21:29:13.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:13 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:13.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:13 smithi083 ceph-mon[24790]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:14.515 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-15T21:29:14.516 INFO:teuthology.orchestra.run.smithi139.stdout:{"epoch":1,"fsid":"e4a42038-fb6e-11ee-bc8f-c7b262605968","modified":"2024-04-15T21:27:34.832030Z","created":"2024-04-15T21:27:34.832030Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-15T21:29:14.517 INFO:teuthology.orchestra.run.smithi139.stderr:dumped monmap epoch 1 2024-04-15T21:29:14.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:14 smithi083 ceph-mon[24790]: from='client.? 172.21.15.139:0/1099453946' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:15.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:15 smithi083 ceph-mon[24790]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:16.171 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-15T21:29:16.172 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph mon dump -f json 2024-04-15T21:29:17.599 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:17 smithi083 ceph-mon[24790]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:17.979 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-15T21:29:17.980 INFO:teuthology.orchestra.run.smithi139.stdout:{"epoch":1,"fsid":"e4a42038-fb6e-11ee-bc8f-c7b262605968","modified":"2024-04-15T21:27:34.832030Z","created":"2024-04-15T21:27:34.832030Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-15T21:29:17.982 INFO:teuthology.orchestra.run.smithi139.stderr:dumped monmap epoch 1 2024-04-15T21:29:19.011 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:18 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:19.011 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:18 smithi083 ceph-mon[24790]: Deploying daemon node-exporter.smithi083 on smithi083 2024-04-15T21:29:19.011 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:18 smithi083 ceph-mon[24790]: from='client.? 172.21.15.139:0/3154306909' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:19.011 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:18 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:19.580 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-15T21:29:19.580 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph mon dump -f json 2024-04-15T21:29:20.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:19 smithi083 ceph-mon[24790]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:21.318 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-15T21:29:21.318 INFO:teuthology.orchestra.run.smithi139.stdout:{"epoch":1,"fsid":"e4a42038-fb6e-11ee-bc8f-c7b262605968","modified":"2024-04-15T21:27:34.832030Z","created":"2024-04-15T21:27:34.832030Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-15T21:29:21.320 INFO:teuthology.orchestra.run.smithi139.stderr:dumped monmap epoch 1 2024-04-15T21:29:22.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:21 smithi083 ceph-mon[24790]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:22.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:21 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:22.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:21 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-15T21:29:22.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:21 smithi083 ceph-mon[24790]: from='client.? 172.21.15.139:0/3778773501' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:22.942 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-15T21:29:22.942 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph mon dump -f json 2024-04-15T21:29:22.952 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:22 smithi083 ceph-mon[24790]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-15T21:29:22.952 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:22 smithi083 ceph-mon[24790]: mgrmap e14: smithi083.kxqarn(active, since 44s) 2024-04-15T21:29:24.705 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-15T21:29:24.706 INFO:teuthology.orchestra.run.smithi139.stdout:{"epoch":1,"fsid":"e4a42038-fb6e-11ee-bc8f-c7b262605968","modified":"2024-04-15T21:27:34.832030Z","created":"2024-04-15T21:27:34.832030Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-15T21:29:24.707 INFO:teuthology.orchestra.run.smithi139.stderr:dumped monmap epoch 1 2024-04-15T21:29:25.043 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:24 smithi083 ceph-mon[24790]: from='client.? 172.21.15.139:0/1886139620' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:26.301 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-15T21:29:26.302 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph mon dump -f json 2024-04-15T21:29:28.124 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-15T21:29:28.124 INFO:teuthology.orchestra.run.smithi139.stdout:{"epoch":1,"fsid":"e4a42038-fb6e-11ee-bc8f-c7b262605968","modified":"2024-04-15T21:27:34.832030Z","created":"2024-04-15T21:27:34.832030Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-15T21:29:28.126 INFO:teuthology.orchestra.run.smithi139.stderr:dumped monmap epoch 1 2024-04-15T21:29:28.825 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:28 smithi083 ceph-mon[24790]: Active manager daemon smithi083.kxqarn restarted 2024-04-15T21:29:28.825 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:28 smithi083 ceph-mon[24790]: Activating manager daemon smithi083.kxqarn 2024-04-15T21:29:28.825 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:28 smithi083 ceph-mon[24790]: osdmap e5: 0 total, 0 up, 0 in 2024-04-15T21:29:28.825 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:28 smithi083 ceph-mon[24790]: from='client.? 172.21.15.139:0/830743193' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:29.656 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-15T21:29:29.656 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph mon dump -f json 2024-04-15T21:29:29.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:29 smithi083 ceph-mon[24790]: mgrmap e15: smithi083.kxqarn(active, starting, since 0.972939s) 2024-04-15T21:29:29.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-04-15T21:29:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mgr metadata", "who": "smithi083.kxqarn", "id": "smithi083.kxqarn"}]: dispatch 2024-04-15T21:29:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-15T21:29:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-15T21:29:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-15T21:29:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:29 smithi083 ceph-mon[24790]: Manager daemon smithi083.kxqarn is now available 2024-04-15T21:29:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:29:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:29:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:29:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:29:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:29:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:29:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:30.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:30 smithi083 ceph-mon[24790]: mgrmap e16: smithi083.kxqarn(active, since 1.9796s) 2024-04-15T21:29:30.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:30 smithi083 ceph-mon[24790]: [15/Apr/2024:21:29:29] ENGINE Bus STARTING 2024-04-15T21:29:30.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:30 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:30.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:30 smithi083 ceph-mon[24790]: [15/Apr/2024:21:29:29] ENGINE Serving on https://172.21.15.83:7150 2024-04-15T21:29:30.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:30 smithi083 ceph-mon[24790]: [15/Apr/2024:21:29:29] ENGINE Bus STARTED 2024-04-15T21:29:30.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:30 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:31.665 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-15T21:29:31.665 INFO:teuthology.orchestra.run.smithi139.stdout:{"epoch":1,"fsid":"e4a42038-fb6e-11ee-bc8f-c7b262605968","modified":"2024-04-15T21:27:34.832030Z","created":"2024-04-15T21:27:34.832030Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-15T21:29:31.666 INFO:teuthology.orchestra.run.smithi139.stderr:dumped monmap epoch 1 2024-04-15T21:29:31.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:31 smithi083 ceph-mon[24790]: mgrmap e17: smithi083.kxqarn(active, since 2s) 2024-04-15T21:29:31.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:31 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:32.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:32 smithi083 ceph-mon[24790]: from='client.? 172.21.15.139:0/364471101' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:33.966 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-15T21:29:33.966 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph mon dump -f json 2024-04-15T21:29:36.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:35 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:36.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:35 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:36.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:35 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:29:36.239 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-15T21:29:36.239 INFO:teuthology.orchestra.run.smithi139.stdout:{"epoch":1,"fsid":"e4a42038-fb6e-11ee-bc8f-c7b262605968","modified":"2024-04-15T21:27:34.832030Z","created":"2024-04-15T21:27:34.832030Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-15T21:29:36.241 INFO:teuthology.orchestra.run.smithi139.stderr:dumped monmap epoch 1 2024-04-15T21:29:37.016 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:36 smithi083 ceph-mon[24790]: from='client.? 172.21.15.139:0/2151704631' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:37.818 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-15T21:29:37.818 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph mon dump -f json 2024-04-15T21:29:39.314 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:39.314 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:39.314 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd/host:smithi083", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:29:39.633 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-15T21:29:39.633 INFO:teuthology.orchestra.run.smithi139.stdout:{"epoch":1,"fsid":"e4a42038-fb6e-11ee-bc8f-c7b262605968","modified":"2024-04-15T21:27:34.832030Z","created":"2024-04-15T21:27:34.832030Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-15T21:29:39.636 INFO:teuthology.orchestra.run.smithi139.stderr:dumped monmap epoch 1 2024-04-15T21:29:40.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:40 smithi083 ceph-mon[24790]: Updating smithi083:/etc/ceph/ceph.conf 2024-04-15T21:29:40.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:40 smithi083 ceph-mon[24790]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-04-15T21:29:40.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:40 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:40.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:40 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:40.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:40 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi139", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-15T21:29:40.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:40 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi139", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-15T21:29:40.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:40 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:29:40.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:40 smithi083 ceph-mon[24790]: Deploying daemon crash.smithi139 on smithi139 2024-04-15T21:29:40.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:40 smithi083 ceph-mon[24790]: from='client.? 172.21.15.139:0/2268981412' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:42.408 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-15T21:29:42.408 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph mon dump -f json 2024-04-15T21:29:43.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:43 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:43.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:43 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi139.bqqykn", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-15T21:29:43.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:43 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi139.bqqykn", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-15T21:29:43.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:43 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-15T21:29:43.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:43 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:29:43.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:43 smithi083 ceph-mon[24790]: Deploying daemon mgr.smithi139.bqqykn on smithi139 2024-04-15T21:29:44.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:44 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:45.245 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-15T21:29:45.245 INFO:teuthology.orchestra.run.smithi139.stdout:{"epoch":1,"fsid":"e4a42038-fb6e-11ee-bc8f-c7b262605968","modified":"2024-04-15T21:27:34.832030Z","created":"2024-04-15T21:27:34.832030Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-15T21:29:45.246 INFO:teuthology.orchestra.run.smithi139.stderr:dumped monmap epoch 1 2024-04-15T21:29:45.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:45 smithi083 ceph-mon[24790]: from='client.? 172.21.15.139:0/4266316492' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:45.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:45 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:45.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:45 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-15T21:29:45.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:45 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:29:46.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:46 smithi083 ceph-mon[24790]: Deploying daemon mon.smithi139 on smithi139 2024-04-15T21:29:47.141 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-15T21:29:47.141 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph mon dump -f json 2024-04-15T21:29:49.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:49 smithi083 ceph-mon[24790]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:49.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:49 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.095 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-15T21:29:50.095 INFO:teuthology.orchestra.run.smithi139.stdout:{"epoch":1,"fsid":"e4a42038-fb6e-11ee-bc8f-c7b262605968","modified":"2024-04-15T21:27:34.832030Z","created":"2024-04-15T21:27:34.832030Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-15T21:29:50.095 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 podman[26882]: 2024-04-15 21:29:50.054056678 +0000 UTC m=+0.121127721 container create 3e6950ee7383f3fb51a25a3a89a74ff68994598286712be292d7b2d6d065f996 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi139, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, architecture=x86_64, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, com.redhat.component=centos-stream-container, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=) 2024-04-15T21:29:50.097 INFO:teuthology.orchestra.run.smithi139.stderr:dumped monmap epoch 1 2024-04-15T21:29:50.833 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 podman[26882]: 2024-04-15 21:29:50.466312955 +0000 UTC m=+0.533384006 container init 3e6950ee7383f3fb51a25a3a89a74ff68994598286712be292d7b2d6d065f996 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi139, ceph=True, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, architecture=x86_64, name=centos-stream, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, version=8, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8) 2024-04-15T21:29:50.834 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 podman[26882]: 2024-04-15 21:29:50.469945144 +0000 UTC m=+0.537016190 container start 3e6950ee7383f3fb51a25a3a89a74ff68994598286712be292d7b2d6d065f996 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi139, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, vcs-type=git, ceph=True, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-15T21:29:50.834 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: set uid:gid to 167:167 (ceph:ceph) 2024-04-15T21:29:50.834 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-15T21:29:50.834 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: pidfile_write: ignore empty --pid-file 2024-04-15T21:29:50.834 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: load: jerasure load: lrc 2024-04-15T21:29:50.834 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: RocksDB version: 6.15.5 2024-04-15T21:29:50.834 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-15T21:29:50.834 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Compile date Apr 18 2022 2024-04-15T21:29:50.834 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: DB SUMMARY 2024-04-15T21:29:50.834 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: DB Session ID: EU45UWCA3CMRQHYSIEZ5 2024-04-15T21:29:50.834 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: CURRENT file: CURRENT 2024-04-15T21:29:50.835 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: IDENTITY file: IDENTITY 2024-04-15T21:29:50.835 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-15T21:29:50.835 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi139/store.db dir, Total Num: 0, files: 2024-04-15T21:29:50.835 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi139/store.db: 000004.log size: 511 ; 2024-04-15T21:29:50.835 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.error_if_exists: 0 2024-04-15T21:29:50.835 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.create_if_missing: 0 2024-04-15T21:29:50.835 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.paranoid_checks: 1 2024-04-15T21:29:50.835 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-15T21:29:50.835 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.env: 0x558544cc6860 2024-04-15T21:29:50.835 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.fs: Posix File System 2024-04-15T21:29:50.835 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.info_log: 0x5585474d9fa0 2024-04-15T21:29:50.835 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_file_opening_threads: 16 2024-04-15T21:29:50.835 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.statistics: (nil) 2024-04-15T21:29:50.835 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.use_fsync: 0 2024-04-15T21:29:50.836 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_log_file_size: 0 2024-04-15T21:29:50.836 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-15T21:29:50.836 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-15T21:29:50.836 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.keep_log_file_num: 1000 2024-04-15T21:29:50.836 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.recycle_log_file_num: 0 2024-04-15T21:29:50.836 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.allow_fallocate: 1 2024-04-15T21:29:50.836 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.allow_mmap_reads: 0 2024-04-15T21:29:50.836 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.allow_mmap_writes: 0 2024-04-15T21:29:50.836 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.use_direct_reads: 0 2024-04-15T21:29:50.836 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-15T21:29:50.836 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.create_missing_column_families: 0 2024-04-15T21:29:50.836 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.db_log_dir: 2024-04-15T21:29:50.836 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi139/store.db 2024-04-15T21:29:50.836 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-15T21:29:50.836 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-15T21:29:50.837 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-15T21:29:50.837 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-15T21:29:50.837 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-15T21:29:50.837 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-15T21:29:50.838 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.advise_random_on_open: 1 2024-04-15T21:29:50.838 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.db_write_buffer_size: 0 2024-04-15T21:29:50.839 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.write_buffer_manager: 0x5585475ca330 2024-04-15T21:29:50.839 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-15T21:29:50.839 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-15T21:29:50.839 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-15T21:29:50.839 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-15T21:29:50.839 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.rate_limiter: (nil) 2024-04-15T21:29:50.839 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-15T21:29:50.839 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.wal_recovery_mode: 2 2024-04-15T21:29:50.839 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.enable_thread_tracking: 0 2024-04-15T21:29:50.839 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.enable_pipelined_write: 0 2024-04-15T21:29:50.839 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.unordered_write: 0 2024-04-15T21:29:50.839 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-15T21:29:50.839 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-15T21:29:50.839 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-15T21:29:50.840 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-15T21:29:50.840 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.row_cache: None 2024-04-15T21:29:50.840 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.wal_filter: None 2024-04-15T21:29:50.840 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-15T21:29:50.840 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.allow_ingest_behind: 0 2024-04-15T21:29:50.840 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.preserve_deletes: 0 2024-04-15T21:29:50.840 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.two_write_queues: 0 2024-04-15T21:29:50.840 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.manual_wal_flush: 0 2024-04-15T21:29:50.840 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.atomic_flush: 0 2024-04-15T21:29:50.840 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-15T21:29:50.840 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-15T21:29:50.840 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-15T21:29:50.840 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.log_readahead_size: 0 2024-04-15T21:29:50.840 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-15T21:29:50.841 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.best_efforts_recovery: 0 2024-04-15T21:29:50.841 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-15T21:29:50.841 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-15T21:29:50.841 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.allow_data_in_errors: 0 2024-04-15T21:29:50.841 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.db_host_id: __hostname__ 2024-04-15T21:29:50.841 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_background_jobs: 2 2024-04-15T21:29:50.841 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_background_compactions: -1 2024-04-15T21:29:50.841 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_subcompactions: 1 2024-04-15T21:29:50.841 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-15T21:29:50.841 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-15T21:29:50.841 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-15T21:29:50.841 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_total_wal_size: 0 2024-04-15T21:29:50.841 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-15T21:29:50.841 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-15T21:29:50.841 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-15T21:29:50.842 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-15T21:29:50.842 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_open_files: -1 2024-04-15T21:29:50.842 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.bytes_per_sync: 0 2024-04-15T21:29:50.842 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-15T21:29:50.842 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-15T21:29:50.842 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compaction_readahead_size: 0 2024-04-15T21:29:50.842 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_background_flushes: -1 2024-04-15T21:29:50.842 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Compression algorithms supported: 2024-04-15T21:29:50.842 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-15T21:29:50.842 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: kZSTD supported: 0 2024-04-15T21:29:50.842 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: kXpressCompression supported: 0 2024-04-15T21:29:50.842 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: kLZ4HCCompression supported: 1 2024-04-15T21:29:50.842 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: kLZ4Compression supported: 1 2024-04-15T21:29:50.842 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: kBZip2Compression supported: 0 2024-04-15T21:29:50.843 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: kZlibCompression supported: 1 2024-04-15T21:29:50.843 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: kSnappyCompression supported: 1 2024-04-15T21:29:50.843 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-15T21:29:50.843 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi139/store.db/MANIFEST-000003 2024-04-15T21:29:50.843 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-15T21:29:50.843 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-15T21:29:50.843 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.merge_operator: 2024-04-15T21:29:50.843 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compaction_filter: None 2024-04-15T21:29:50.843 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compaction_filter_factory: None 2024-04-15T21:29:50.843 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.sst_partitioner_factory: None 2024-04-15T21:29:50.843 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-15T21:29:50.843 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-15T21:29:50.843 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5585474a5d00) 2024-04-15T21:29:50.843 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: cache_index_and_filter_blocks: 1 2024-04-15T21:29:50.844 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-15T21:29:50.844 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-15T21:29:50.844 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: pin_top_level_index_and_filter: 1 2024-04-15T21:29:50.844 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: index_type: 0 2024-04-15T21:29:50.844 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: data_block_index_type: 0 2024-04-15T21:29:50.844 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: index_shortening: 1 2024-04-15T21:29:50.844 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-15T21:29:50.844 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: hash_index_allow_collision: 1 2024-04-15T21:29:50.844 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: checksum: 1 2024-04-15T21:29:50.844 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: no_block_cache: 0 2024-04-15T21:29:50.844 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: block_cache: 0x558547510170 2024-04-15T21:29:50.844 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: block_cache_name: BinnedLRUCache 2024-04-15T21:29:50.844 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: block_cache_options: 2024-04-15T21:29:50.844 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: capacity : 536870912 2024-04-15T21:29:50.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: num_shard_bits : 4 2024-04-15T21:29:50.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: strict_capacity_limit : 0 2024-04-15T21:29:50.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: high_pri_pool_ratio: 0.000 2024-04-15T21:29:50.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: block_cache_compressed: (nil) 2024-04-15T21:29:50.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: persistent_cache: (nil) 2024-04-15T21:29:50.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: block_size: 4096 2024-04-15T21:29:50.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: block_size_deviation: 10 2024-04-15T21:29:50.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: block_restart_interval: 16 2024-04-15T21:29:50.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: index_block_restart_interval: 1 2024-04-15T21:29:50.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: metadata_block_size: 4096 2024-04-15T21:29:50.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: partition_filters: 0 2024-04-15T21:29:50.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: use_delta_encoding: 1 2024-04-15T21:29:50.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-15T21:29:50.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: whole_key_filtering: 1 2024-04-15T21:29:50.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: verify_compression: 0 2024-04-15T21:29:50.846 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: read_amp_bytes_per_bit: 0 2024-04-15T21:29:50.846 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: format_version: 4 2024-04-15T21:29:50.846 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: enable_index_compression: 1 2024-04-15T21:29:50.846 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: block_align: 0 2024-04-15T21:29:50.846 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.write_buffer_size: 33554432 2024-04-15T21:29:50.846 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_write_buffer_number: 2 2024-04-15T21:29:50.846 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compression: NoCompression 2024-04-15T21:29:50.846 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.bottommost_compression: Disabled 2024-04-15T21:29:50.846 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.prefix_extractor: nullptr 2024-04-15T21:29:50.846 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-15T21:29:50.846 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.num_levels: 7 2024-04-15T21:29:50.846 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-15T21:29:50.846 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-15T21:29:50.846 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-15T21:29:50.847 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-15T21:29:50.847 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-15T21:29:50.847 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-15T21:29:50.847 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-15T21:29:50.847 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-15T21:29:50.847 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-15T21:29:50.847 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-15T21:29:50.847 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-15T21:29:50.847 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compression_opts.level: 32767 2024-04-15T21:29:50.847 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compression_opts.strategy: 0 2024-04-15T21:29:50.847 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-15T21:29:50.847 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-15T21:29:50.847 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-15T21:29:50.847 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compression_opts.enabled: false 2024-04-15T21:29:50.847 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-15T21:29:50.848 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-15T21:29:50.848 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-15T21:29:50.848 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.target_file_size_base: 67108864 2024-04-15T21:29:50.848 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-15T21:29:50.848 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-15T21:29:50.848 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-15T21:29:50.848 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-15T21:29:50.848 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-15T21:29:50.848 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-15T21:29:50.848 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-15T21:29:50.848 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-15T21:29:50.848 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-15T21:29:50.848 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-15T21:29:50.848 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-15T21:29:50.848 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-15T21:29:50.849 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-15T21:29:50.849 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.arena_block_size: 4194304 2024-04-15T21:29:50.849 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-15T21:29:50.849 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-15T21:29:50.849 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-15T21:29:50.849 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.disable_auto_compactions: 0 2024-04-15T21:29:50.849 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-15T21:29:50.849 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-15T21:29:50.849 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-15T21:29:50.849 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-15T21:29:50.849 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-15T21:29:50.849 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-15T21:29:50.849 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-15T21:29:50.849 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-15T21:29:50.849 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-15T21:29:50.850 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-15T21:29:50.850 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.table_properties_collectors: 2024-04-15T21:29:50.850 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.inplace_update_support: 0 2024-04-15T21:29:50.850 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-15T21:29:50.850 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-15T21:29:50.850 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-15T21:29:50.850 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-15T21:29:50.850 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.bloom_locality: 0 2024-04-15T21:29:50.850 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.max_successive_merges: 0 2024-04-15T21:29:50.850 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-15T21:29:50.850 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.paranoid_file_checks: 0 2024-04-15T21:29:50.850 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.force_consistency_checks: 1 2024-04-15T21:29:50.850 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.report_bg_io_stats: 0 2024-04-15T21:29:50.851 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.ttl: 2592000 2024-04-15T21:29:50.851 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-15T21:29:50.851 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.enable_blob_files: false 2024-04-15T21:29:50.851 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.min_blob_size: 0 2024-04-15T21:29:50.851 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.blob_file_size: 268435456 2024-04-15T21:29:50.851 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-15T21:29:50.851 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-15T21:29:50.851 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-15T21:29:50.851 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi139/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-15T21:29:50.851 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-04-15T21:29:50.851 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-04-15T21:29:50.851 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713216590496490, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-15T21:29:50.851 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-04-15T21:29:50.851 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-15T21:29:50.851 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713216590496922, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1713216590, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "676622b4-2717-4b3d-aff5-a8648b91beb3", "db_session_id": "EU45UWCA3CMRQHYSIEZ5"}} 2024-04-15T21:29:50.852 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-04-15T21:29:50.852 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713216590497334, "job": 1, "event": "recovery_finished"} 2024-04-15T21:29:50.852 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi139/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-15T21:29:50.852 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x5585474f6a80 2024-04-15T21:29:50.852 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: DB pointer 0x558547506000 2024-04-15T21:29:50.852 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-15T21:29:50.852 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-15T21:29:50.852 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: ** DB Stats ** 2024-04-15T21:29:50.852 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-15T21:29:50.852 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-15T21:29:50.852 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-15T21:29:50.852 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-15T21:29:50.852 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-15T21:29:50.852 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-15T21:29:50.852 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-15T21:29:50.853 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: 2024-04-15T21:29:50.853 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: ** Compaction Stats [default] ** 2024-04-15T21:29:50.853 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-15T21:29:50.853 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-15T21:29:50.853 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-15T21:29:50.853 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-15T21:29:50.853 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-15T21:29:50.853 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: 2024-04-15T21:29:50.853 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: ** Compaction Stats [default] ** 2024-04-15T21:29:50.853 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-15T21:29:50.853 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-15T21:29:50.853 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-15T21:29:50.853 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-15T21:29:50.853 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-15T21:29:50.854 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-15T21:29:50.854 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-15T21:29:50.854 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-15T21:29:50.854 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-15T21:29:50.854 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Cumulative compaction: 0.00 GB write, 0.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-15T21:29:50.854 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Interval compaction: 0.00 GB write, 0.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-15T21:29:50.854 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-15T21:29:50.854 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: 2024-04-15T21:29:50.854 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-15T21:29:50.854 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: 2024-04-15T21:29:50.854 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: ** Compaction Stats [default] ** 2024-04-15T21:29:50.854 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-15T21:29:50.854 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-15T21:29:50.854 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-15T21:29:50.855 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-15T21:29:50.855 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-15T21:29:50.855 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: 2024-04-15T21:29:50.855 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: ** Compaction Stats [default] ** 2024-04-15T21:29:50.855 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-15T21:29:50.855 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-15T21:29:50.855 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-15T21:29:50.855 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-15T21:29:50.855 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-15T21:29:50.855 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-15T21:29:50.855 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-15T21:29:50.855 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-15T21:29:50.855 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-15T21:29:50.855 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Cumulative compaction: 0.00 GB write, 0.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-15T21:29:50.856 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-15T21:29:50.856 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-15T21:29:50.856 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: 2024-04-15T21:29:50.856 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-15T21:29:50.856 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mon.smithi139 does not exist in monmap, will attempt to join an existing cluster 2024-04-15T21:29:50.856 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: using public_addr v2:172.21.15.139:0/0 -> [v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0] 2024-04-15T21:29:50.856 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: starting mon.smithi139 rank -1 at public addrs [v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0] at bind addrs [v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi139 fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 2024-04-15T21:29:50.856 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mon.smithi139@-1(???) e0 preinit fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 2024-04-15T21:29:50.856 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mon.smithi139@-1(synchronizing).mds e1 new map 2024-04-15T21:29:50.856 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mon.smithi139@-1(synchronizing).mds e1 print_map 2024-04-15T21:29:50.856 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: e1 2024-04-15T21:29:50.856 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-15T21:29:50.856 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-15T21:29:50.856 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: legacy client fscid: -1 2024-04-15T21:29:50.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: 2024-04-15T21:29:50.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout: No filesystems configured 2024-04-15T21:29:50.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mon.smithi139@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-15T21:29:50.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mon.smithi139@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-15T21:29:50.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mon.smithi139@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-15T21:29:50.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mon.smithi139@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-15T21:29:50.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mon.smithi139@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-15T21:29:50.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mon.smithi139@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-15T21:29:50.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mon.smithi139@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-04-15T21:29:50.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mon.smithi139@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-15T21:29:50.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mon.smithi139@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-04-15T21:29:50.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mon.smithi139@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-04-15T21:29:50.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mon.smithi139@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-04-15T21:29:50.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: Deploying daemon crash.smithi083 on smithi083 2024-04-15T21:29:50.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: Deploying cephadm binary to smithi139 2024-04-15T21:29:50.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: Added host smithi139 2024-04-15T21:29:50.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:50.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:50.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-15T21:29:50.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-15T21:29:50.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: Deploying daemon grafana.smithi083 on smithi083 2024-04-15T21:29:50.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-15T21:29:50.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:50.859 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:50.859 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.859 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/588203402' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-15T21:29:50.859 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/588203402' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-15T21:29:50.859 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: osdmap e4: 0 total, 0 up, 0 in 2024-04-15T21:29:50.859 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:50.859 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi083:172.21.15.83=smithi083;smithi139:172.21.15.139=smithi139", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:29:50.859 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: Saving service mon spec with placement smithi083:172.21.15.83=smithi083;smithi139:172.21.15.139=smithi139;count:2 2024-04-15T21:29:50.859 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.859 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:50.859 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='client.? 172.21.15.139:0/1099453946' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:50.859 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:50.859 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:50.859 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.860 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: Deploying daemon node-exporter.smithi083 on smithi083 2024-04-15T21:29:50.860 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='client.? 172.21.15.139:0/3154306909' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:50.860 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.860 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:50.860 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:50.860 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.860 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-15T21:29:50.860 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='client.? 172.21.15.139:0/3778773501' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:50.860 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14160 172.21.15.83:0/3078772023' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-15T21:29:50.860 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mgrmap e14: smithi083.kxqarn(active, since 44s) 2024-04-15T21:29:50.860 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='client.? 172.21.15.139:0/1886139620' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:50.860 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: Active manager daemon smithi083.kxqarn restarted 2024-04-15T21:29:50.860 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: Activating manager daemon smithi083.kxqarn 2024-04-15T21:29:50.860 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: osdmap e5: 0 total, 0 up, 0 in 2024-04-15T21:29:50.860 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='client.? 172.21.15.139:0/830743193' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:50.861 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mgrmap e15: smithi083.kxqarn(active, starting, since 0.972939s) 2024-04-15T21:29:50.861 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-04-15T21:29:50.861 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mgr metadata", "who": "smithi083.kxqarn", "id": "smithi083.kxqarn"}]: dispatch 2024-04-15T21:29:50.861 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-15T21:29:50.861 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-15T21:29:50.861 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-15T21:29:50.861 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: Manager daemon smithi083.kxqarn is now available 2024-04-15T21:29:50.861 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:29:50.861 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:29:50.861 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:29:50.861 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:29:50.861 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.861 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:29:50.861 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:29:50.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mgrmap e16: smithi083.kxqarn(active, since 1.9796s) 2024-04-15T21:29:50.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: [15/Apr/2024:21:29:29] ENGINE Bus STARTING 2024-04-15T21:29:50.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: [15/Apr/2024:21:29:29] ENGINE Serving on https://172.21.15.83:7150 2024-04-15T21:29:50.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: [15/Apr/2024:21:29:29] ENGINE Bus STARTED 2024-04-15T21:29:50.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mgrmap e17: smithi083.kxqarn(active, since 2s) 2024-04-15T21:29:50.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='client.? 172.21.15.139:0/364471101' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:50.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:29:50.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='client.? 172.21.15.139:0/2151704631' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:50.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd/host:smithi083", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:29:50.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: Updating smithi083:/etc/ceph/ceph.conf 2024-04-15T21:29:50.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-04-15T21:29:50.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi139", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-15T21:29:50.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi139", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-15T21:29:50.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:29:50.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: Deploying daemon crash.smithi139 on smithi139 2024-04-15T21:29:50.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='client.? 172.21.15.139:0/2268981412' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:50.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi139.bqqykn", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-15T21:29:50.864 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi139.bqqykn", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-15T21:29:50.864 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-15T21:29:50.864 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:29:50.864 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: Deploying daemon mgr.smithi139.bqqykn on smithi139 2024-04-15T21:29:50.864 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.864 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='client.? 172.21.15.139:0/4266316492' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:50.864 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.864 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-15T21:29:50.864 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:29:50.864 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: Deploying daemon mon.smithi139 on smithi139 2024-04-15T21:29:50.864 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:50.864 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:50.864 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mon.smithi139@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-04-15T21:29:50.864 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 ceph-mon[26919]: mon.smithi139@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-04-15T21:29:50.864 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 bash[26882]: 3e6950ee7383f3fb51a25a3a89a74ff68994598286712be292d7b2d6d065f996 2024-04-15T21:29:50.864 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:50 smithi139 systemd[1]: Started Ceph mon.smithi139 for e4a42038-fb6e-11ee-bc8f-c7b262605968. 2024-04-15T21:29:52.027 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-15T21:29:52.027 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph mon dump -f json 2024-04-15T21:29:55.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:55.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-04-15T21:29:55.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: mon.smithi083 calling monitor election 2024-04-15T21:29:55.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi139"}]: dispatch 2024-04-15T21:29:55.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: from='mgr.? 172.21.15.139:0/2667861083' entity='mgr.smithi139.bqqykn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi139.bqqykn/crt"}]: dispatch 2024-04-15T21:29:55.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi139"}]: dispatch 2024-04-15T21:29:55.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:55.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi139"}]: dispatch 2024-04-15T21:29:55.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: mon.smithi139 calling monitor election 2024-04-15T21:29:55.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi139"}]: dispatch 2024-04-15T21:29:55.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:55.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi139"}]: dispatch 2024-04-15T21:29:55.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi139"}]: dispatch 2024-04-15T21:29:55.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: mon.smithi083 is new leader, mons smithi083,smithi139 in quorum (ranks 0,1) 2024-04-15T21:29:55.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: monmap e2: 2 mons at {smithi083=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],smithi139=[v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0]} 2024-04-15T21:29:55.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: fsmap 2024-04-15T21:29:55.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: osdmap e5: 0 total, 0 up, 0 in 2024-04-15T21:29:55.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: mgrmap e17: smithi083.kxqarn(active, since 28s) 2024-04-15T21:29:55.852 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: Standby manager daemon smithi139.bqqykn started 2024-04-15T21:29:55.852 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: from='mgr.? 172.21.15.139:0/2667861083' entity='mgr.smithi139.bqqykn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-15T21:29:55.852 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: overall HEALTH_OK 2024-04-15T21:29:55.852 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: from='mgr.? 172.21.15.139:0/2667861083' entity='mgr.smithi139.bqqykn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi139.bqqykn/key"}]: dispatch 2024-04-15T21:29:55.852 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:55.852 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:55 smithi083 ceph-mon[24790]: from='mgr.? 172.21.15.139:0/2667861083' entity='mgr.smithi139.bqqykn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-15T21:29:55.856 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:55.856 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-04-15T21:29:55.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: mon.smithi083 calling monitor election 2024-04-15T21:29:55.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi139"}]: dispatch 2024-04-15T21:29:55.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: from='mgr.? 172.21.15.139:0/2667861083' entity='mgr.smithi139.bqqykn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi139.bqqykn/crt"}]: dispatch 2024-04-15T21:29:55.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi139"}]: dispatch 2024-04-15T21:29:55.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:55.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi139"}]: dispatch 2024-04-15T21:29:55.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: mon.smithi139 calling monitor election 2024-04-15T21:29:55.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi139"}]: dispatch 2024-04-15T21:29:55.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:55.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi139"}]: dispatch 2024-04-15T21:29:55.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi139"}]: dispatch 2024-04-15T21:29:55.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: mon.smithi083 is new leader, mons smithi083,smithi139 in quorum (ranks 0,1) 2024-04-15T21:29:55.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: monmap e2: 2 mons at {smithi083=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],smithi139=[v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0]} 2024-04-15T21:29:55.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: fsmap 2024-04-15T21:29:55.857 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: osdmap e5: 0 total, 0 up, 0 in 2024-04-15T21:29:55.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: mgrmap e17: smithi083.kxqarn(active, since 28s) 2024-04-15T21:29:55.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: Standby manager daemon smithi139.bqqykn started 2024-04-15T21:29:55.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: from='mgr.? 172.21.15.139:0/2667861083' entity='mgr.smithi139.bqqykn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-15T21:29:55.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: overall HEALTH_OK 2024-04-15T21:29:55.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: from='mgr.? 172.21.15.139:0/2667861083' entity='mgr.smithi139.bqqykn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi139.bqqykn/key"}]: dispatch 2024-04-15T21:29:55.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:55.858 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:55 smithi139 ceph-mon[26919]: from='mgr.? 172.21.15.139:0/2667861083' entity='mgr.smithi139.bqqykn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-15T21:29:56.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:56 smithi083 ceph-mon[24790]: Deploying daemon node-exporter.smithi139 on smithi139 2024-04-15T21:29:56.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:56 smithi083 ceph-mon[24790]: mgrmap e18: smithi083.kxqarn(active, since 28s), standbys: smithi139.bqqykn 2024-04-15T21:29:56.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:56 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mgr metadata", "who": "smithi139.bqqykn", "id": "smithi139.bqqykn"}]: dispatch 2024-04-15T21:29:56.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:56 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi139"}]: dispatch 2024-04-15T21:29:56.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:56 smithi139 ceph-mon[26919]: Deploying daemon node-exporter.smithi139 on smithi139 2024-04-15T21:29:56.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:56 smithi139 ceph-mon[26919]: mgrmap e18: smithi083.kxqarn(active, since 28s), standbys: smithi139.bqqykn 2024-04-15T21:29:56.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:56 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mgr metadata", "who": "smithi139.bqqykn", "id": "smithi139.bqqykn"}]: dispatch 2024-04-15T21:29:56.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:56 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi139"}]: dispatch 2024-04-15T21:29:56.998 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-15T21:29:56.998 INFO:teuthology.orchestra.run.smithi139.stdout:{"epoch":2,"fsid":"e4a42038-fb6e-11ee-bc8f-c7b262605968","modified":"2024-04-15T21:29:50.523423Z","created":"2024-04-15T21:27:34.832030Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi139","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:3300","nonce":0},{"type":"v1","addr":"172.21.15.139:6789","nonce":0}]},"addr":"172.21.15.139:6789/0","public_addr":"172.21.15.139:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-15T21:29:56.999 INFO:teuthology.orchestra.run.smithi139.stderr:dumped monmap epoch 2 2024-04-15T21:29:57.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:57 smithi083 ceph-mon[24790]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:57.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:57 smithi083 ceph-mon[24790]: from='client.? 172.21.15.139:0/1817231390' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:57.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:57 smithi139 ceph-mon[26919]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:57.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:57 smithi139 ceph-mon[26919]: from='client.? 172.21.15.139:0/1817231390' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-15T21:29:58.160 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-15T21:29:58.161 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph config generate-minimal-conf 2024-04-15T21:29:59.591 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:59 smithi083 ceph-mon[24790]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:59.592 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:29:59 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:29:59.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:59 smithi139 ceph-mon[26919]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:29:59.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:29:59 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:00.026 INFO:teuthology.orchestra.run.smithi083.stdout:# minimal ceph.conf for e4a42038-fb6e-11ee-bc8f-c7b262605968 2024-04-15T21:30:00.026 INFO:teuthology.orchestra.run.smithi083.stdout:[global] 2024-04-15T21:30:00.026 INFO:teuthology.orchestra.run.smithi083.stdout: fsid = e4a42038-fb6e-11ee-bc8f-c7b262605968 2024-04-15T21:30:00.026 INFO:teuthology.orchestra.run.smithi083.stdout: mon_host = [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] [v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0] 2024-04-15T21:30:00.723 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-15T21:30:00.723 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-15T21:30:00.723 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/ceph/ceph.conf 2024-04-15T21:30:00.749 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:00 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:00.749 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:00 smithi083 ceph-mon[24790]: Deploying daemon prometheus.smithi083 on smithi083 2024-04-15T21:30:00.749 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:00 smithi083 ceph-mon[24790]: overall HEALTH_OK 2024-04-15T21:30:00.749 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:00 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/408698591' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:00.753 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-15T21:30:00.753 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-15T21:30:00.822 DEBUG:teuthology.orchestra.run.smithi139:> set -ex 2024-04-15T21:30:00.822 DEBUG:teuthology.orchestra.run.smithi139:> sudo dd of=/etc/ceph/ceph.conf 2024-04-15T21:30:00.853 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:00 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:00.853 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:00 smithi139 ceph-mon[26919]: Deploying daemon prometheus.smithi083 on smithi083 2024-04-15T21:30:00.853 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:00 smithi139 ceph-mon[26919]: overall HEALTH_OK 2024-04-15T21:30:00.853 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:00 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/408698591' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:00.859 DEBUG:teuthology.orchestra.run.smithi139:> set -ex 2024-04-15T21:30:00.859 DEBUG:teuthology.orchestra.run.smithi139:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-15T21:30:00.931 INFO:tasks.cephadm:Deploying OSDs... 2024-04-15T21:30:00.931 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-15T21:30:00.931 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/scratch_devs of=/dev/stdout 2024-04-15T21:30:00.947 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-15T21:30:00.947 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_1 2024-04-15T21:30:01.003 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-15T21:30:01.003 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-15T21:30:01.003 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 663 Links: 1 2024-04-15T21:30:01.003 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-15T21:30:01.003 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-04-15T21:30:01.003 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-04-15 21:29:18.376937779 +0000 2024-04-15T21:30:01.003 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-04-15 21:26:37.030914356 +0000 2024-04-15T21:30:01.003 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-04-15 21:26:37.030914356 +0000 2024-04-15T21:30:01.003 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-04-15 21:26:37.030914356 +0000 2024-04-15T21:30:01.004 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-15T21:30:01.070 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-04-15T21:30:01.070 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-04-15T21:30:01.070 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000264823 s, 1.9 MB/s 2024-04-15T21:30:01.071 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-15T21:30:01.128 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_2 2024-04-15T21:30:01.184 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-15T21:30:01.184 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-15T21:30:01.184 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 680 Links: 1 2024-04-15T21:30:01.184 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-15T21:30:01.184 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-04-15T21:30:01.185 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-04-15 21:29:18.376937779 +0000 2024-04-15T21:30:01.185 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-04-15 21:26:37.371906218 +0000 2024-04-15T21:30:01.185 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-04-15 21:26:37.371906218 +0000 2024-04-15T21:30:01.185 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-04-15 21:26:37.371906218 +0000 2024-04-15T21:30:01.185 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-15T21:30:01.251 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-04-15T21:30:01.251 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-04-15T21:30:01.251 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000211733 s, 2.4 MB/s 2024-04-15T21:30:01.252 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-15T21:30:01.309 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_3 2024-04-15T21:30:01.363 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-15T21:30:01.363 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-15T21:30:01.363 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 698 Links: 1 2024-04-15T21:30:01.363 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-15T21:30:01.364 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-04-15T21:30:01.364 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-04-15 21:29:18.377937755 +0000 2024-04-15T21:30:01.364 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-04-15 21:26:37.717897962 +0000 2024-04-15T21:30:01.364 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-04-15 21:26:37.717897962 +0000 2024-04-15T21:30:01.364 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-04-15 21:26:37.717897962 +0000 2024-04-15T21:30:01.364 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-15T21:30:01.431 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-04-15T21:30:01.431 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-04-15T21:30:01.431 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000205018 s, 2.5 MB/s 2024-04-15T21:30:01.433 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-15T21:30:01.489 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_4 2024-04-15T21:30:01.544 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-15T21:30:01.544 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-15T21:30:01.544 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 719 Links: 1 2024-04-15T21:30:01.545 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-15T21:30:01.545 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-04-15T21:30:01.545 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-04-15 21:29:18.377937755 +0000 2024-04-15T21:30:01.545 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-04-15 21:26:38.056889872 +0000 2024-04-15T21:30:01.545 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-04-15 21:26:38.056889872 +0000 2024-04-15T21:30:01.545 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-04-15 21:26:38.056889872 +0000 2024-04-15T21:30:01.545 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-15T21:30:01.612 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-04-15T21:30:01.612 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-04-15T21:30:01.612 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000170382 s, 3.0 MB/s 2024-04-15T21:30:01.614 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-15T21:30:01.669 DEBUG:teuthology.orchestra.run.smithi139:> set -ex 2024-04-15T21:30:01.670 DEBUG:teuthology.orchestra.run.smithi139:> dd if=/scratch_devs of=/dev/stdout 2024-04-15T21:30:01.690 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-15T21:30:01.690 DEBUG:teuthology.orchestra.run.smithi139:> stat /dev/vg_nvme/lv_1 2024-04-15T21:30:01.748 INFO:teuthology.orchestra.run.smithi139.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-15T21:30:01.748 INFO:teuthology.orchestra.run.smithi139.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-15T21:30:01.748 INFO:teuthology.orchestra.run.smithi139.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-15T21:30:01.749 INFO:teuthology.orchestra.run.smithi139.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-15T21:30:01.749 INFO:teuthology.orchestra.run.smithi139.stdout:Context: system_u:object_r:device_t:s0 2024-04-15T21:30:01.749 INFO:teuthology.orchestra.run.smithi139.stdout:Access: 2024-04-15 21:29:56.141919593 +0000 2024-04-15T21:30:01.749 INFO:teuthology.orchestra.run.smithi139.stdout:Modify: 2024-04-15 21:26:30.676240796 +0000 2024-04-15T21:30:01.749 INFO:teuthology.orchestra.run.smithi139.stdout:Change: 2024-04-15 21:26:30.676240796 +0000 2024-04-15T21:30:01.749 INFO:teuthology.orchestra.run.smithi139.stdout: Birth: 2024-04-15 21:26:30.676240796 +0000 2024-04-15T21:30:01.749 DEBUG:teuthology.orchestra.run.smithi139:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-15T21:30:01.819 INFO:teuthology.orchestra.run.smithi139.stderr:1+0 records in 2024-04-15T21:30:01.819 INFO:teuthology.orchestra.run.smithi139.stderr:1+0 records out 2024-04-15T21:30:01.819 INFO:teuthology.orchestra.run.smithi139.stderr:512 bytes copied, 0.000235259 s, 2.2 MB/s 2024-04-15T21:30:01.819 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:01 smithi139 ceph-mon[26919]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:01.820 DEBUG:teuthology.orchestra.run.smithi139:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-15T21:30:01.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:01 smithi083 ceph-mon[24790]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:01.879 DEBUG:teuthology.orchestra.run.smithi139:> stat /dev/vg_nvme/lv_2 2024-04-15T21:30:01.936 INFO:teuthology.orchestra.run.smithi139.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-15T21:30:01.936 INFO:teuthology.orchestra.run.smithi139.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-15T21:30:01.936 INFO:teuthology.orchestra.run.smithi139.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-04-15T21:30:01.937 INFO:teuthology.orchestra.run.smithi139.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-15T21:30:01.937 INFO:teuthology.orchestra.run.smithi139.stdout:Context: system_u:object_r:device_t:s0 2024-04-15T21:30:01.937 INFO:teuthology.orchestra.run.smithi139.stdout:Access: 2024-04-15 21:29:56.141919593 +0000 2024-04-15T21:30:01.937 INFO:teuthology.orchestra.run.smithi139.stdout:Modify: 2024-04-15 21:26:31.026233723 +0000 2024-04-15T21:30:01.937 INFO:teuthology.orchestra.run.smithi139.stdout:Change: 2024-04-15 21:26:31.026233723 +0000 2024-04-15T21:30:01.937 INFO:teuthology.orchestra.run.smithi139.stdout: Birth: 2024-04-15 21:26:31.026233723 +0000 2024-04-15T21:30:01.937 DEBUG:teuthology.orchestra.run.smithi139:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-15T21:30:02.005 INFO:teuthology.orchestra.run.smithi139.stderr:1+0 records in 2024-04-15T21:30:02.005 INFO:teuthology.orchestra.run.smithi139.stderr:1+0 records out 2024-04-15T21:30:02.005 INFO:teuthology.orchestra.run.smithi139.stderr:512 bytes copied, 0.000237547 s, 2.2 MB/s 2024-04-15T21:30:02.006 DEBUG:teuthology.orchestra.run.smithi139:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-15T21:30:02.063 DEBUG:teuthology.orchestra.run.smithi139:> stat /dev/vg_nvme/lv_3 2024-04-15T21:30:02.119 INFO:teuthology.orchestra.run.smithi139.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-15T21:30:02.119 INFO:teuthology.orchestra.run.smithi139.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-15T21:30:02.119 INFO:teuthology.orchestra.run.smithi139.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-04-15T21:30:02.119 INFO:teuthology.orchestra.run.smithi139.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-15T21:30:02.120 INFO:teuthology.orchestra.run.smithi139.stdout:Context: system_u:object_r:device_t:s0 2024-04-15T21:30:02.120 INFO:teuthology.orchestra.run.smithi139.stdout:Access: 2024-04-15 21:29:56.141919593 +0000 2024-04-15T21:30:02.120 INFO:teuthology.orchestra.run.smithi139.stdout:Modify: 2024-04-15 21:26:31.349227195 +0000 2024-04-15T21:30:02.120 INFO:teuthology.orchestra.run.smithi139.stdout:Change: 2024-04-15 21:26:31.349227195 +0000 2024-04-15T21:30:02.120 INFO:teuthology.orchestra.run.smithi139.stdout: Birth: 2024-04-15 21:26:31.349227195 +0000 2024-04-15T21:30:02.120 DEBUG:teuthology.orchestra.run.smithi139:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-15T21:30:02.186 INFO:teuthology.orchestra.run.smithi139.stderr:1+0 records in 2024-04-15T21:30:02.187 INFO:teuthology.orchestra.run.smithi139.stderr:1+0 records out 2024-04-15T21:30:02.187 INFO:teuthology.orchestra.run.smithi139.stderr:512 bytes copied, 0.000205713 s, 2.5 MB/s 2024-04-15T21:30:02.188 DEBUG:teuthology.orchestra.run.smithi139:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-15T21:30:02.245 DEBUG:teuthology.orchestra.run.smithi139:> stat /dev/vg_nvme/lv_4 2024-04-15T21:30:02.301 INFO:teuthology.orchestra.run.smithi139.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-15T21:30:02.302 INFO:teuthology.orchestra.run.smithi139.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-15T21:30:02.302 INFO:teuthology.orchestra.run.smithi139.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-04-15T21:30:02.302 INFO:teuthology.orchestra.run.smithi139.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-15T21:30:02.302 INFO:teuthology.orchestra.run.smithi139.stdout:Context: system_u:object_r:device_t:s0 2024-04-15T21:30:02.302 INFO:teuthology.orchestra.run.smithi139.stdout:Access: 2024-04-15 21:29:56.142919574 +0000 2024-04-15T21:30:02.302 INFO:teuthology.orchestra.run.smithi139.stdout:Modify: 2024-04-15 21:26:31.683220444 +0000 2024-04-15T21:30:02.302 INFO:teuthology.orchestra.run.smithi139.stdout:Change: 2024-04-15 21:26:31.683220444 +0000 2024-04-15T21:30:02.302 INFO:teuthology.orchestra.run.smithi139.stdout: Birth: 2024-04-15 21:26:31.683220444 +0000 2024-04-15T21:30:02.302 DEBUG:teuthology.orchestra.run.smithi139:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-15T21:30:02.369 INFO:teuthology.orchestra.run.smithi139.stderr:1+0 records in 2024-04-15T21:30:02.369 INFO:teuthology.orchestra.run.smithi139.stderr:1+0 records out 2024-04-15T21:30:02.369 INFO:teuthology.orchestra.run.smithi139.stderr:512 bytes copied, 0.00025255 s, 2.0 MB/s 2024-04-15T21:30:02.370 DEBUG:teuthology.orchestra.run.smithi139:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-15T21:30:02.427 INFO:tasks.cephadm:Deploying osd.0 on smithi083 with /dev/vg_nvme/lv_4... 2024-04-15T21:30:02.428 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-15T21:30:03.793 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:03 smithi083 ceph-mon[24790]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:03.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:03 smithi139 ceph-mon[26919]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:04.756 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:04 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:04.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:04 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:05.161 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-15T21:30:05.161 INFO:teuthology.orchestra.run.smithi083.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-15T21:30:05.161 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10+0 records in 2024-04-15T21:30:05.161 INFO:teuthology.orchestra.run.smithi083.stderr:10+0 records out 2024-04-15T21:30:05.161 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0119571 s, 877 MB/s 2024-04-15T21:30:05.161 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping successful for: 2024-04-15T21:30:05.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:05 smithi083 ceph-mon[24790]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:05.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:05 smithi139 ceph-mon[26919]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:06.917 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph orch daemon add osd smithi083:vg_nvme/lv_4 2024-04-15T21:30:07.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:07 smithi083 ceph-mon[24790]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:07.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:07 smithi139 ceph-mon[26919]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:09.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:09 smithi083 ceph-mon[24790]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:09.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:09 smithi139 ceph-mon[26919]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:10.684 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:10 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-15T21:30:10.684 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:10 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-15T21:30:10.684 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:10 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:10.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:10 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-15T21:30:10.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:10 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-15T21:30:10.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:10 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:11.707 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:11 smithi083 ceph-mon[24790]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:30:11.707 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:11 smithi083 ceph-mon[24790]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:11.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:11 smithi139 ceph-mon[26919]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:30:11.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:11 smithi139 ceph-mon[26919]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:13.518 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:13 smithi083 ceph-mon[24790]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:13.767 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:13 smithi139 ceph-mon[26919]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:14.934 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:14 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:14.934 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:14 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:30:14.934 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:14 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:14.934 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:14 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:30:15.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:14 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:15.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:14 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:30:15.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:14 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:15.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:14 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:30:15.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:15 smithi083 ceph-mon[24790]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:15.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:15 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/3844287691' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4384484b-f651-428f-a059-88f15e28d2d4"}]: dispatch 2024-04-15T21:30:15.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:15 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/3844287691' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4384484b-f651-428f-a059-88f15e28d2d4"}]': finished 2024-04-15T21:30:15.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:15 smithi083 ceph-mon[24790]: osdmap e6: 1 total, 0 up, 1 in 2024-04-15T21:30:15.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:15 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-15T21:30:15.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:15 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:15.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:15 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/2146125742' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-15T21:30:15.895 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:15 smithi139 ceph-mon[26919]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:15.895 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:15 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/3844287691' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4384484b-f651-428f-a059-88f15e28d2d4"}]: dispatch 2024-04-15T21:30:15.895 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:15 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/3844287691' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4384484b-f651-428f-a059-88f15e28d2d4"}]': finished 2024-04-15T21:30:15.895 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:15 smithi139 ceph-mon[26919]: osdmap e6: 1 total, 0 up, 1 in 2024-04-15T21:30:15.895 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:15 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-15T21:30:15.895 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:15 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:15.895 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:15 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/2146125742' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-15T21:30:17.055 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:16 smithi083 ceph-mon[24790]: Updating smithi139:/etc/ceph/ceph.conf 2024-04-15T21:30:17.055 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:16 smithi083 ceph-mon[24790]: Updating smithi139:/etc/ceph/ceph.client.admin.keyring 2024-04-15T21:30:17.055 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:17.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:16 smithi139 ceph-mon[26919]: Updating smithi139:/etc/ceph/ceph.conf 2024-04-15T21:30:17.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:16 smithi139 ceph-mon[26919]: Updating smithi139:/etc/ceph/ceph.client.admin.keyring 2024-04-15T21:30:17.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:18.014 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:17 smithi139 ceph-mon[26919]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:18.018 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:17 smithi083 ceph-mon[24790]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:19.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:19.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:19 smithi083 ceph-mon[24790]: Updating smithi083:/etc/ceph/ceph.conf 2024-04-15T21:30:19.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:19.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:19.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:19 smithi083 ceph-mon[24790]: Reconfiguring mon.smithi083 (unknown last config time)... 2024-04-15T21:30:19.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-15T21:30:19.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-15T21:30:19.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:19.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:19 smithi083 ceph-mon[24790]: Reconfiguring daemon mon.smithi083 on smithi083 2024-04-15T21:30:19.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:19 smithi083 ceph-mon[24790]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:19.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:19.362 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:19.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:19 smithi139 ceph-mon[26919]: Updating smithi083:/etc/ceph/ceph.conf 2024-04-15T21:30:19.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:19.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:19.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:19 smithi139 ceph-mon[26919]: Reconfiguring mon.smithi083 (unknown last config time)... 2024-04-15T21:30:19.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-15T21:30:19.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-15T21:30:19.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:19.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:19 smithi139 ceph-mon[26919]: Reconfiguring daemon mon.smithi083 on smithi083 2024-04-15T21:30:19.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:19 smithi139 ceph-mon[26919]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:19.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:21.564 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:21 smithi083 ceph-mon[24790]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:21.564 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:21 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:21.564 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:21 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi083.kxqarn", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-15T21:30:21.565 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:21 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-15T21:30:21.565 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:21 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:21.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:21 smithi139 ceph-mon[26919]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:21.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:21 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:21.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:21 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi083.kxqarn", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-15T21:30:21.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:21 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-15T21:30:21.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:21 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:22.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:22 smithi083 ceph-mon[24790]: Reconfiguring mgr.smithi083.kxqarn (unknown last config time)... 2024-04-15T21:30:22.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:22 smithi083 ceph-mon[24790]: Reconfiguring daemon mgr.smithi083.kxqarn on smithi083 2024-04-15T21:30:22.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:22 smithi139 ceph-mon[26919]: Reconfiguring mgr.smithi083.kxqarn (unknown last config time)... 2024-04-15T21:30:22.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:22 smithi139 ceph-mon[26919]: Reconfiguring daemon mgr.smithi083.kxqarn on smithi083 2024-04-15T21:30:23.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:23 smithi083 ceph-mon[24790]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:23.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:23 smithi139 ceph-mon[26919]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:24.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:24 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-15T21:30:24.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:24 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:24.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:24 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-15T21:30:24.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:24 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:25.730 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:25 smithi083 ceph-mon[24790]: Deploying daemon osd.0 on smithi083 2024-04-15T21:30:25.730 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:25 smithi083 ceph-mon[24790]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:25.730 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:25 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:25.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:25 smithi139 ceph-mon[26919]: Deploying daemon osd.0 on smithi083 2024-04-15T21:30:25.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:25 smithi139 ceph-mon[26919]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:25.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:25 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:26.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:26 smithi083 ceph-mon[24790]: Reconfiguring alertmanager.smithi083 (dependencies changed)... 2024-04-15T21:30:26.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:26 smithi083 ceph-mon[24790]: Reconfiguring daemon alertmanager.smithi083 on smithi083 2024-04-15T21:30:26.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:26 smithi139 ceph-mon[26919]: Reconfiguring alertmanager.smithi083 (dependencies changed)... 2024-04-15T21:30:26.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:26 smithi139 ceph-mon[26919]: Reconfiguring daemon alertmanager.smithi083 on smithi083 2024-04-15T21:30:27.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:27 smithi083 ceph-mon[24790]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:27.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:27 smithi139 ceph-mon[26919]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:28.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:30:28.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:30:28.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:30:28.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:30:29.847 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:29 smithi083 ceph-mon[24790]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:29.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:29 smithi139 ceph-mon[26919]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:31.350 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:31 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:31.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:31 smithi083 ceph-mon[24790]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:31.362 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:31 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:31.362 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:31 smithi139 ceph-mon[26919]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:33.839 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:33 smithi083 ceph-mon[24790]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:33.839 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:33 smithi083 ceph-mon[24790]: from='osd.0 [v2:172.21.15.83:6802/1609792529,v1:172.21.15.83:6803/1609792529]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-15T21:30:33.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:33 smithi139 ceph-mon[26919]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:33.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:33 smithi139 ceph-mon[26919]: from='osd.0 [v2:172.21.15.83:6802/1609792529,v1:172.21.15.83:6803/1609792529]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-15T21:30:34.588 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:34 smithi083 ceph-mon[24790]: from='osd.0 [v2:172.21.15.83:6802/1609792529,v1:172.21.15.83:6803/1609792529]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-15T21:30:34.588 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:34 smithi083 ceph-mon[24790]: osdmap e7: 1 total, 0 up, 1 in 2024-04-15T21:30:34.588 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:34 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-15T21:30:34.589 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:34 smithi083 ceph-mon[24790]: from='osd.0 [v2:172.21.15.83:6802/1609792529,v1:172.21.15.83:6803/1609792529]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-15T21:30:34.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:34 smithi139 ceph-mon[26919]: from='osd.0 [v2:172.21.15.83:6802/1609792529,v1:172.21.15.83:6803/1609792529]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-15T21:30:34.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:34 smithi139 ceph-mon[26919]: osdmap e7: 1 total, 0 up, 1 in 2024-04-15T21:30:34.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:34 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-15T21:30:34.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:34 smithi139 ceph-mon[26919]: from='osd.0 [v2:172.21.15.83:6802/1609792529,v1:172.21.15.83:6803/1609792529]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-15T21:30:35.560 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:35 smithi083 ceph-mon[24790]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:35.561 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:35 smithi083 ceph-mon[24790]: from='osd.0 [v2:172.21.15.83:6802/1609792529,v1:172.21.15.83:6803/1609792529]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-15T21:30:35.561 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:35 smithi083 ceph-mon[24790]: osdmap e8: 1 total, 0 up, 1 in 2024-04-15T21:30:35.561 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:35 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-15T21:30:35.561 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:35 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-15T21:30:35.561 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:35 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:35.561 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:35 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-15T21:30:35.561 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:35 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:35.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:35 smithi139 ceph-mon[26919]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:35.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:35 smithi139 ceph-mon[26919]: from='osd.0 [v2:172.21.15.83:6802/1609792529,v1:172.21.15.83:6803/1609792529]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-15T21:30:35.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:35 smithi139 ceph-mon[26919]: osdmap e8: 1 total, 0 up, 1 in 2024-04-15T21:30:35.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:35 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-15T21:30:35.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:35 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-15T21:30:35.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:35 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:35.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:35 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-15T21:30:35.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:35 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:36.775 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:36 smithi083 ceph-mon[24790]: purged_snaps scrub starts 2024-04-15T21:30:36.775 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:36 smithi083 ceph-mon[24790]: purged_snaps scrub ok 2024-04-15T21:30:36.775 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:36 smithi083 ceph-mon[24790]: Reconfiguring crash.smithi083 (monmap changed)... 2024-04-15T21:30:36.775 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:36 smithi083 ceph-mon[24790]: Reconfiguring daemon crash.smithi083 on smithi083 2024-04-15T21:30:36.775 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:36 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-15T21:30:36.776 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:36 smithi083 ceph-mon[24790]: osd.0 [v2:172.21.15.83:6802/1609792529,v1:172.21.15.83:6803/1609792529] boot 2024-04-15T21:30:36.776 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:36 smithi083 ceph-mon[24790]: osdmap e9: 1 total, 1 up, 1 in 2024-04-15T21:30:36.776 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:36 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-15T21:30:36.809 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 0 on host 'smithi083' 2024-04-15T21:30:36.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:36 smithi139 ceph-mon[26919]: purged_snaps scrub starts 2024-04-15T21:30:36.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:36 smithi139 ceph-mon[26919]: purged_snaps scrub ok 2024-04-15T21:30:36.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:36 smithi139 ceph-mon[26919]: Reconfiguring crash.smithi083 (monmap changed)... 2024-04-15T21:30:36.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:36 smithi139 ceph-mon[26919]: Reconfiguring daemon crash.smithi083 on smithi083 2024-04-15T21:30:36.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:36 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-15T21:30:36.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:36 smithi139 ceph-mon[26919]: osd.0 [v2:172.21.15.83:6802/1609792529,v1:172.21.15.83:6803/1609792529] boot 2024-04-15T21:30:36.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:36 smithi139 ceph-mon[26919]: osdmap e9: 1 total, 1 up, 1 in 2024-04-15T21:30:36.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:36 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-15T21:30:37.782 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:37 smithi083 ceph-mon[24790]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:37.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:37 smithi083 ceph-mon[24790]: osdmap e10: 1 total, 1 up, 1 in 2024-04-15T21:30:37.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:37 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:37.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:37 smithi139 ceph-mon[26919]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-15T21:30:37.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:37 smithi139 ceph-mon[26919]: osdmap e10: 1 total, 1 up, 1 in 2024-04-15T21:30:37.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:37 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:38.533 DEBUG:teuthology.orchestra.run.smithi083:osd.0> sudo journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.0.service 2024-04-15T21:30:38.536 INFO:tasks.cephadm:Deploying osd.1 on smithi083 with /dev/vg_nvme/lv_3... 2024-04-15T21:30:38.536 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-15T21:30:39.584 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:39.584 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:39 smithi083 ceph-mon[24790]: Reconfiguring grafana.smithi083 (dependencies changed)... 2024-04-15T21:30:39.585 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:39 smithi083 ceph-mon[24790]: Reconfiguring daemon grafana.smithi083 on smithi083 2024-04-15T21:30:39.585 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:39 smithi083 ceph-mon[24790]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:39.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:39.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:39 smithi139 ceph-mon[26919]: Reconfiguring grafana.smithi083 (dependencies changed)... 2024-04-15T21:30:39.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:39 smithi139 ceph-mon[26919]: Reconfiguring daemon grafana.smithi083 on smithi083 2024-04-15T21:30:39.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:39 smithi139 ceph-mon[26919]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:41.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:41 smithi083 ceph-mon[24790]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:41.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:41 smithi139 ceph-mon[26919]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:42.930 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-15T21:30:42.930 INFO:teuthology.orchestra.run.smithi083.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-15T21:30:42.930 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10+0 records in 2024-04-15T21:30:42.930 INFO:teuthology.orchestra.run.smithi083.stderr:10+0 records out 2024-04-15T21:30:42.930 INFO:teuthology.orchestra.run.smithi083.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0123393 s, 850 MB/s 2024-04-15T21:30:42.930 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping successful for: 2024-04-15T21:30:43.524 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:43 smithi083 ceph-mon[24790]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:43.549 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph orch daemon add osd smithi083:vg_nvme/lv_3 2024-04-15T21:30:43.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:43 smithi139 ceph-mon[26919]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:45.350 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:45 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:45.350 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:45 smithi083 ceph-mon[24790]: Reconfiguring crash.smithi139 (monmap changed)... 2024-04-15T21:30:45.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:45 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi139", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-15T21:30:45.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:45 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:45.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:45 smithi083 ceph-mon[24790]: Reconfiguring daemon crash.smithi139 on smithi139 2024-04-15T21:30:45.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:45 smithi083 ceph-mon[24790]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:45.362 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:45 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:45.362 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:45 smithi139 ceph-mon[26919]: Reconfiguring crash.smithi139 (monmap changed)... 2024-04-15T21:30:45.362 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:45 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi139", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-15T21:30:45.362 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:45 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:45.362 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:45 smithi139 ceph-mon[26919]: Reconfiguring daemon crash.smithi139 on smithi139 2024-04-15T21:30:45.362 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:45 smithi139 ceph-mon[26919]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:46.585 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:46 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:46.585 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:46 smithi083 ceph-mon[24790]: Reconfiguring mgr.smithi139.bqqykn (monmap changed)... 2024-04-15T21:30:46.586 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:46 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi139.bqqykn", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-15T21:30:46.586 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:46 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-15T21:30:46.586 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:46 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:46.586 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:46 smithi083 ceph-mon[24790]: Reconfiguring daemon mgr.smithi139.bqqykn on smithi139 2024-04-15T21:30:46.586 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:46 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-15T21:30:46.586 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:46 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-15T21:30:46.586 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:46 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:46.783 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:46 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:46.784 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:46 smithi139 ceph-mon[26919]: Reconfiguring mgr.smithi139.bqqykn (monmap changed)... 2024-04-15T21:30:46.784 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:46 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi139.bqqykn", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-15T21:30:46.784 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:46 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-15T21:30:46.784 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:46 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:46.784 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:46 smithi139 ceph-mon[26919]: Reconfiguring daemon mgr.smithi139.bqqykn on smithi139 2024-04-15T21:30:46.784 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:46 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-15T21:30:46.784 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:46 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-15T21:30:46.784 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:46 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:47.507 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:47 smithi083 ceph-mon[24790]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:30:47.507 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:47 smithi083 ceph-mon[24790]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:47.507 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:47 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:47.507 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:47 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-15T21:30:47.507 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:47 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-15T21:30:47.508 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:47 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:47.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:47 smithi139 ceph-mon[26919]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:30:47.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:47 smithi139 ceph-mon[26919]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:47.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:47 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:47.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:47 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-15T21:30:47.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:47 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-15T21:30:47.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:47 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:48.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:48 smithi083 ceph-mon[24790]: Reconfiguring mon.smithi139 (monmap changed)... 2024-04-15T21:30:48.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:48 smithi083 ceph-mon[24790]: Reconfiguring daemon mon.smithi139 on smithi139 2024-04-15T21:30:48.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:48 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:48.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:48 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-15T21:30:48.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:48 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/304246187' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "886cd293-b724-4daa-81c2-aa6d6dc9821a"}]: dispatch 2024-04-15T21:30:48.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:48 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/304246187' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "886cd293-b724-4daa-81c2-aa6d6dc9821a"}]': finished 2024-04-15T21:30:48.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:48 smithi083 ceph-mon[24790]: osdmap e11: 2 total, 1 up, 2 in 2024-04-15T21:30:48.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:48 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-15T21:30:48.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:48 smithi139 ceph-mon[26919]: Reconfiguring mon.smithi139 (monmap changed)... 2024-04-15T21:30:48.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:48 smithi139 ceph-mon[26919]: Reconfiguring daemon mon.smithi139 on smithi139 2024-04-15T21:30:48.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:48 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:48.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:48 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-15T21:30:48.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:48 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/304246187' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "886cd293-b724-4daa-81c2-aa6d6dc9821a"}]: dispatch 2024-04-15T21:30:48.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:48 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/304246187' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "886cd293-b724-4daa-81c2-aa6d6dc9821a"}]': finished 2024-04-15T21:30:48.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:48 smithi139 ceph-mon[26919]: osdmap e11: 2 total, 1 up, 2 in 2024-04-15T21:30:48.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:48 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-15T21:30:49.751 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:49 smithi083 ceph-mon[24790]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-15T21:30:49.751 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:49 smithi083 ceph-mon[24790]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:49.752 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:49 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/3317239723' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-15T21:30:49.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:49 smithi139 ceph-mon[26919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-15T21:30:49.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:49 smithi139 ceph-mon[26919]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:49.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:49 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/3317239723' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-15T21:30:51.674 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:51 smithi083 ceph-mon[24790]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:51.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:51 smithi139 ceph-mon[26919]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:53.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:53 smithi083 ceph-mon[24790]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:53.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:53 smithi139 ceph-mon[26919]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:54.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:54 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-15T21:30:54.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:54 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:54.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:54 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-15T21:30:54.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:54 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:30:55.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:55 smithi083 ceph-mon[24790]: Deploying daemon osd.1 on smithi083 2024-04-15T21:30:55.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:55 smithi083 ceph-mon[24790]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:55.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:55 smithi139 ceph-mon[26919]: Deploying daemon osd.1 on smithi083 2024-04-15T21:30:55.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:55 smithi139 ceph-mon[26919]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:57.827 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:57 smithi083 ceph-mon[24790]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:57.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:57 smithi139 ceph-mon[26919]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:59.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:59 smithi083 ceph-mon[24790]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:59.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:30:59 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:30:59.844 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:59 smithi139 ceph-mon[26919]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:30:59.844 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:30:59 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:01.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:01 smithi083 ceph-mon[24790]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:31:01.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:01 smithi139 ceph-mon[26919]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:31:02.286 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 1 on host 'smithi083' 2024-04-15T21:31:02.962 DEBUG:teuthology.orchestra.run.smithi083:osd.1> sudo journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.1.service 2024-04-15T21:31:02.964 INFO:tasks.cephadm:Deploying osd.2 on smithi083 with /dev/vg_nvme/lv_2... 2024-04-15T21:31:02.965 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-15T21:31:03.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:03 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:03.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:03 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.83:9093"}]: dispatch 2024-04-15T21:31:03.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:03 smithi083 ceph-mon[24790]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.83:9093"}]: dispatch 2024-04-15T21:31:03.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:03 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:03.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:03 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-15T21:31:03.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:03 smithi083 ceph-mon[24790]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-15T21:31:03.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:03 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.83:3000"}]: dispatch 2024-04-15T21:31:03.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:03 smithi083 ceph-mon[24790]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.83:3000"}]: dispatch 2024-04-15T21:31:03.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:03 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:03.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:03 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-15T21:31:03.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:03 smithi083 ceph-mon[24790]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-15T21:31:03.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:03 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.83:9095"}]: dispatch 2024-04-15T21:31:03.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:03 smithi083 ceph-mon[24790]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.83:9095"}]: dispatch 2024-04-15T21:31:03.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:03 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:03.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:03 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:31:03.602 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:03 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:31:03.602 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:03 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:31:03.602 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:03 smithi083 ceph-mon[24790]: from='osd.1 [v2:172.21.15.83:6810/3642014361,v1:172.21.15.83:6811/3642014361]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-15T21:31:03.602 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:03 smithi083 ceph-mon[24790]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:31:03.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:03 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:03.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:03 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.83:9093"}]: dispatch 2024-04-15T21:31:03.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:03 smithi139 ceph-mon[26919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.83:9093"}]: dispatch 2024-04-15T21:31:03.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:03 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:03.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:03 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-15T21:31:03.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:03 smithi139 ceph-mon[26919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-15T21:31:03.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:03 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.83:3000"}]: dispatch 2024-04-15T21:31:03.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:03 smithi139 ceph-mon[26919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.83:3000"}]: dispatch 2024-04-15T21:31:03.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:03 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:03.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:03 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-15T21:31:03.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:03 smithi139 ceph-mon[26919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-15T21:31:03.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:03 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.83:9095"}]: dispatch 2024-04-15T21:31:03.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:03 smithi139 ceph-mon[26919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.83:9095"}]: dispatch 2024-04-15T21:31:03.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:03 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:03.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:03 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:31:03.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:03 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:31:03.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:03 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:31:03.614 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:03 smithi139 ceph-mon[26919]: from='osd.1 [v2:172.21.15.83:6810/3642014361,v1:172.21.15.83:6811/3642014361]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-15T21:31:03.614 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:03 smithi139 ceph-mon[26919]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:31:04.326 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:04 smithi139 ceph-mon[26919]: from='osd.1 [v2:172.21.15.83:6810/3642014361,v1:172.21.15.83:6811/3642014361]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-15T21:31:04.327 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:04 smithi139 ceph-mon[26919]: osdmap e12: 2 total, 1 up, 2 in 2024-04-15T21:31:04.327 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:04 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-15T21:31:04.327 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:04 smithi139 ceph-mon[26919]: from='osd.1 [v2:172.21.15.83:6810/3642014361,v1:172.21.15.83:6811/3642014361]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-15T21:31:04.350 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:04 smithi083 ceph-mon[24790]: from='osd.1 [v2:172.21.15.83:6810/3642014361,v1:172.21.15.83:6811/3642014361]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-15T21:31:04.350 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:04 smithi083 ceph-mon[24790]: osdmap e12: 2 total, 1 up, 2 in 2024-04-15T21:31:04.350 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:04 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-15T21:31:04.350 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:04 smithi083 ceph-mon[24790]: from='osd.1 [v2:172.21.15.83:6810/3642014361,v1:172.21.15.83:6811/3642014361]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-15T21:31:04.351 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:31:04 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-1[38036]: 2024-04-15T21:31:04.326+0000 7f7ddfb73700 -1 osd.1 0 waiting for initial osdmap 2024-04-15T21:31:04.351 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:31:04 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-1[38036]: 2024-04-15T21:31:04.329+0000 7f7dd8506700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-15T21:31:05.350 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:05 smithi083 ceph-mon[24790]: from='osd.1 [v2:172.21.15.83:6810/3642014361,v1:172.21.15.83:6811/3642014361]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-15T21:31:05.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:05 smithi083 ceph-mon[24790]: osdmap e13: 2 total, 1 up, 2 in 2024-04-15T21:31:05.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:05 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-15T21:31:05.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:05 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-15T21:31:05.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:05 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:05.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:05 smithi083 ceph-mon[24790]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:31:05.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:05 smithi139 ceph-mon[26919]: from='osd.1 [v2:172.21.15.83:6810/3642014361,v1:172.21.15.83:6811/3642014361]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-15T21:31:05.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:05 smithi139 ceph-mon[26919]: osdmap e13: 2 total, 1 up, 2 in 2024-04-15T21:31:05.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:05 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-15T21:31:05.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:05 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-15T21:31:05.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:05 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:05.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:05 smithi139 ceph-mon[26919]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-15T21:31:05.831 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-15T21:31:05.831 INFO:teuthology.orchestra.run.smithi083.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-15T21:31:05.831 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10+0 records in 2024-04-15T21:31:05.831 INFO:teuthology.orchestra.run.smithi083.stderr:10+0 records out 2024-04-15T21:31:05.831 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0122622 s, 855 MB/s 2024-04-15T21:31:05.831 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping successful for: 2024-04-15T21:31:06.405 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:06 smithi083 ceph-mon[24790]: purged_snaps scrub starts 2024-04-15T21:31:06.405 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:06 smithi083 ceph-mon[24790]: purged_snaps scrub ok 2024-04-15T21:31:06.405 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:06 smithi083 ceph-mon[24790]: osd.1 [v2:172.21.15.83:6810/3642014361,v1:172.21.15.83:6811/3642014361] boot 2024-04-15T21:31:06.405 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:06 smithi083 ceph-mon[24790]: osdmap e14: 2 total, 2 up, 2 in 2024-04-15T21:31:06.405 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:06 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-15T21:31:06.407 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph orch daemon add osd smithi083:vg_nvme/lv_2 2024-04-15T21:31:06.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:06 smithi139 ceph-mon[26919]: purged_snaps scrub starts 2024-04-15T21:31:06.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:06 smithi139 ceph-mon[26919]: purged_snaps scrub ok 2024-04-15T21:31:06.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:06 smithi139 ceph-mon[26919]: osd.1 [v2:172.21.15.83:6810/3642014361,v1:172.21.15.83:6811/3642014361] boot 2024-04-15T21:31:06.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:06 smithi139 ceph-mon[26919]: osdmap e14: 2 total, 2 up, 2 in 2024-04-15T21:31:06.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:06 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-15T21:31:07.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:07 smithi083 ceph-mon[24790]: osdmap e15: 2 total, 2 up, 2 in 2024-04-15T21:31:07.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:07 smithi083 ceph-mon[24790]: pgmap v52: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:07.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:07 smithi139 ceph-mon[26919]: osdmap e15: 2 total, 2 up, 2 in 2024-04-15T21:31:07.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:07 smithi139 ceph-mon[26919]: pgmap v52: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:09.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:09 smithi083 ceph-mon[24790]: pgmap v53: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:09.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:09 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-15T21:31:09.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:09 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-15T21:31:09.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:09 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:31:09.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:09 smithi139 ceph-mon[26919]: pgmap v53: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:09.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:09 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-15T21:31:09.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:09 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-15T21:31:09.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:09 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:31:10.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:10 smithi083 ceph-mon[24790]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:31:10.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:10 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:10.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:10 smithi139 ceph-mon[26919]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:31:10.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:10 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:11.712 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:11 smithi083 ceph-mon[24790]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:11.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:11 smithi139 ceph-mon[26919]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:12.728 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:12 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/557980977' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fddc5d10-da39-4a82-87cc-36e35a7e5601"}]: dispatch 2024-04-15T21:31:12.728 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:12 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/557980977' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fddc5d10-da39-4a82-87cc-36e35a7e5601"}]': finished 2024-04-15T21:31:12.728 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:12 smithi083 ceph-mon[24790]: osdmap e16: 3 total, 2 up, 3 in 2024-04-15T21:31:12.728 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:12 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-15T21:31:12.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:12 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/557980977' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fddc5d10-da39-4a82-87cc-36e35a7e5601"}]: dispatch 2024-04-15T21:31:12.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:12 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/557980977' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fddc5d10-da39-4a82-87cc-36e35a7e5601"}]': finished 2024-04-15T21:31:12.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:12 smithi139 ceph-mon[26919]: osdmap e16: 3 total, 2 up, 3 in 2024-04-15T21:31:12.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:12 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-15T21:31:13.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:13 smithi083 ceph-mon[24790]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:13.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:13 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1153083181' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-15T21:31:13.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:13 smithi139 ceph-mon[26919]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:13.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:13 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/1153083181' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-15T21:31:15.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:15 smithi083 ceph-mon[24790]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:15.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:15 smithi139 ceph-mon[26919]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:17.804 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:17 smithi083 ceph-mon[24790]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:17.807 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:17 smithi139 ceph-mon[26919]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:18.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:18 smithi083 ceph-mon[24790]: Detected new or changed devices on smithi083 2024-04-15T21:31:18.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:18 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:18.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:18 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:31:18.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:18 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:31:18.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:18 smithi083 ceph-mon[24790]: Adjusting osd_memory_target on smithi083 to 6432M 2024-04-15T21:31:18.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:18 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:18.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:18 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:18.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:18 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:31:18.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:18 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:31:18.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:18 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:31:19.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:18 smithi139 ceph-mon[26919]: Detected new or changed devices on smithi083 2024-04-15T21:31:19.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:18 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:19.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:18 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:31:19.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:18 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:31:19.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:18 smithi139 ceph-mon[26919]: Adjusting osd_memory_target on smithi083 to 6432M 2024-04-15T21:31:19.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:18 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:19.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:18 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:19.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:18 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:31:19.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:18 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:31:19.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:18 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:31:19.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:19 smithi139 ceph-mon[26919]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:19.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:20.048 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:19 smithi083 ceph-mon[24790]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:20.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:20.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:20 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-15T21:31:20.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:20 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:31:21.097 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:20 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-15T21:31:21.097 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:20 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:31:22.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:21 smithi083 ceph-mon[24790]: Deploying daemon osd.2 on smithi083 2024-04-15T21:31:22.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:21 smithi083 ceph-mon[24790]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:22.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:21 smithi139 ceph-mon[26919]: Deploying daemon osd.2 on smithi083 2024-04-15T21:31:22.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:21 smithi139 ceph-mon[26919]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:23.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:23 smithi083 ceph-mon[24790]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:23.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:23 smithi139 ceph-mon[26919]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:25.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:25 smithi083 ceph-mon[24790]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:25.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:25 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:25.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:25 smithi139 ceph-mon[26919]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:25.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:25 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:27.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:27 smithi083 ceph-mon[24790]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:27.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:27 smithi139 ceph-mon[26919]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:29.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:29.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:28 smithi139 ceph-mon[26919]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:29.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:31:29.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:31:29.347 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:29.347 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:28 smithi083 ceph-mon[24790]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:29.347 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:31:29.347 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:31:30.957 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:30 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:30.957 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:30 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:30.957 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:30 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:31:30.957 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:30 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:31:30.957 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:30 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:31:30.986 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:30 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:30.986 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:30 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:30.986 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:30 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:31:30.986 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:30 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:31:30.986 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:30 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:31:31.724 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 2 on host 'smithi083' 2024-04-15T21:31:31.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:31 smithi083 ceph-mon[24790]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:31.883 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:31 smithi083 ceph-mon[24790]: from='osd.2 [v2:172.21.15.83:6818/2309901275,v1:172.21.15.83:6819/2309901275]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-15T21:31:31.883 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:31 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:32.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:31 smithi139 ceph-mon[26919]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:32.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:31 smithi139 ceph-mon[26919]: from='osd.2 [v2:172.21.15.83:6818/2309901275,v1:172.21.15.83:6819/2309901275]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-15T21:31:32.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:31 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:32.713 DEBUG:teuthology.orchestra.run.smithi083:osd.2> sudo journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.2.service 2024-04-15T21:31:32.716 INFO:tasks.cephadm:Deploying osd.3 on smithi083 with /dev/vg_nvme/lv_1... 2024-04-15T21:31:32.716 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-15T21:31:33.019 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:32 smithi083 ceph-mon[24790]: from='osd.2 [v2:172.21.15.83:6818/2309901275,v1:172.21.15.83:6819/2309901275]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-15T21:31:33.019 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:32 smithi083 ceph-mon[24790]: osdmap e17: 3 total, 2 up, 3 in 2024-04-15T21:31:33.019 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:32 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-15T21:31:33.020 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:32 smithi083 ceph-mon[24790]: from='osd.2 [v2:172.21.15.83:6818/2309901275,v1:172.21.15.83:6819/2309901275]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-15T21:31:33.020 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:32 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:33.020 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:32 smithi083 ceph-mon[24790]: from='osd.2 [v2:172.21.15.83:6818/2309901275,v1:172.21.15.83:6819/2309901275]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-15T21:31:33.020 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:32 smithi083 ceph-mon[24790]: osdmap e18: 3 total, 2 up, 3 in 2024-04-15T21:31:33.020 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:32 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-15T21:31:33.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:32 smithi139 ceph-mon[26919]: from='osd.2 [v2:172.21.15.83:6818/2309901275,v1:172.21.15.83:6819/2309901275]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-15T21:31:33.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:32 smithi139 ceph-mon[26919]: osdmap e17: 3 total, 2 up, 3 in 2024-04-15T21:31:33.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:32 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-15T21:31:33.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:32 smithi139 ceph-mon[26919]: from='osd.2 [v2:172.21.15.83:6818/2309901275,v1:172.21.15.83:6819/2309901275]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-15T21:31:33.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:32 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:33.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:32 smithi139 ceph-mon[26919]: from='osd.2 [v2:172.21.15.83:6818/2309901275,v1:172.21.15.83:6819/2309901275]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-15T21:31:33.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:32 smithi139 ceph-mon[26919]: osdmap e18: 3 total, 2 up, 3 in 2024-04-15T21:31:33.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:32 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-15T21:31:33.922 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:33 smithi083 ceph-mon[24790]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:33.922 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:33 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-15T21:31:33.922 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:33 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-15T21:31:34.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:33 smithi139 ceph-mon[26919]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:34.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:33 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-15T21:31:34.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:33 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-15T21:31:35.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:34 smithi083 ceph-mon[24790]: purged_snaps scrub starts 2024-04-15T21:31:35.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:34 smithi083 ceph-mon[24790]: purged_snaps scrub ok 2024-04-15T21:31:35.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:34 smithi083 ceph-mon[24790]: osd.2 [v2:172.21.15.83:6818/2309901275,v1:172.21.15.83:6819/2309901275] boot 2024-04-15T21:31:35.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:34 smithi083 ceph-mon[24790]: osdmap e19: 3 total, 3 up, 3 in 2024-04-15T21:31:35.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:34 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-15T21:31:35.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:34 smithi139 ceph-mon[26919]: purged_snaps scrub starts 2024-04-15T21:31:35.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:34 smithi139 ceph-mon[26919]: purged_snaps scrub ok 2024-04-15T21:31:35.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:34 smithi139 ceph-mon[26919]: osd.2 [v2:172.21.15.83:6818/2309901275,v1:172.21.15.83:6819/2309901275] boot 2024-04-15T21:31:35.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:34 smithi139 ceph-mon[26919]: osdmap e19: 3 total, 3 up, 3 in 2024-04-15T21:31:35.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:34 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-15T21:31:35.932 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-15T21:31:35.932 INFO:teuthology.orchestra.run.smithi083.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-15T21:31:35.932 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10+0 records in 2024-04-15T21:31:35.932 INFO:teuthology.orchestra.run.smithi083.stderr:10+0 records out 2024-04-15T21:31:35.932 INFO:teuthology.orchestra.run.smithi083.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0113766 s, 922 MB/s 2024-04-15T21:31:35.932 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping successful for: 2024-04-15T21:31:36.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:35 smithi083 ceph-mon[24790]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:36.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:35 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-15T21:31:36.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:35 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-15T21:31:36.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:35 smithi083 ceph-mon[24790]: osdmap e20: 3 total, 3 up, 3 in 2024-04-15T21:31:36.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:35 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-15T21:31:36.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:35 smithi139 ceph-mon[26919]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-15T21:31:36.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:35 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-15T21:31:36.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:35 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-15T21:31:36.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:35 smithi139 ceph-mon[26919]: osdmap e20: 3 total, 3 up, 3 in 2024-04-15T21:31:36.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:35 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-15T21:31:36.729 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:31:36 smithi083 sudo[43181]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-15T21:31:36.729 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:31:36 smithi083 sudo[43181]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-15T21:31:36.729 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:31:36 smithi083 sudo[43181]: pam_unix(sudo:session): session closed for user root 2024-04-15T21:31:36.948 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph orch daemon add osd smithi083:vg_nvme/lv_1 2024-04-15T21:31:37.002 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:31:36 smithi083 sudo[43202]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-15T21:31:37.002 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:31:36 smithi083 sudo[43202]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-15T21:31:37.002 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:31:36 smithi083 sudo[43202]: pam_unix(sudo:session): session closed for user root 2024-04-15T21:31:37.003 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:36 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-15T21:31:37.003 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:36 smithi083 ceph-mon[24790]: osdmap e21: 3 total, 3 up, 3 in 2024-04-15T21:31:37.003 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:36 smithi083 ceph-mon[24790]: pgmap v73: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-15T21:31:37.003 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:36 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:37.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:36 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-15T21:31:37.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:36 smithi139 ceph-mon[26919]: osdmap e21: 3 total, 3 up, 3 in 2024-04-15T21:31:37.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:36 smithi139 ceph-mon[26919]: pgmap v73: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-15T21:31:37.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:36 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:37.254 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:31:37 smithi083 sudo[43237]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-15T21:31:37.254 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:31:37 smithi083 sudo[43237]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-15T21:31:37.254 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:31:37 smithi083 sudo[43237]: pam_unix(sudo:session): session closed for user root 2024-04-15T21:31:37.522 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:31:37 smithi083 sudo[43278]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-15T21:31:37.522 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:31:37 smithi083 sudo[43278]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-15T21:31:37.522 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:31:37 smithi083 sudo[43278]: pam_unix(sudo:session): session closed for user root 2024-04-15T21:31:37.788 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:31:37 smithi083 sudo[43306]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-15T21:31:37.788 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:31:37 smithi083 sudo[43306]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-15T21:31:37.788 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:31:37 smithi083 sudo[43306]: pam_unix(sudo:session): session closed for user root 2024-04-15T21:31:38.039 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:31:37 smithi083 sudo[43348]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-15T21:31:38.039 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:31:37 smithi083 sudo[43348]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-15T21:31:38.039 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:31:37 smithi083 sudo[43348]: pam_unix(sudo:session): session closed for user root 2024-04-15T21:31:38.310 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:38 smithi083 sudo[43372]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-15T21:31:38.310 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:38 smithi083 sudo[43372]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-15T21:31:38.310 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:38 smithi083 sudo[43372]: pam_unix(sudo:session): session closed for user root 2024-04-15T21:31:38.568 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:38 smithi139 ceph-mon[26919]: osdmap e22: 3 total, 3 up, 3 in 2024-04-15T21:31:38.568 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:38 smithi139 ceph-mon[26919]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-15T21:31:38.568 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:38 smithi139 sudo[29240]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-15T21:31:38.568 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:38 smithi139 sudo[29240]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-15T21:31:38.568 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:38 smithi139 sudo[29240]: pam_unix(sudo:session): session closed for user root 2024-04-15T21:31:38.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:38 smithi083 sudo[43387]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-15T21:31:38.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:38 smithi083 sudo[43387]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-15T21:31:38.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:38 smithi083 sudo[43387]: pam_unix(sudo:session): session closed for user root 2024-04-15T21:31:38.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:38 smithi083 ceph-mon[24790]: osdmap e22: 3 total, 3 up, 3 in 2024-04-15T21:31:38.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:38 smithi083 ceph-mon[24790]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-15T21:31:38.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:38 smithi139 sudo[29243]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-15T21:31:38.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:38 smithi139 sudo[29243]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-15T21:31:38.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:38 smithi139 sudo[29243]: pam_unix(sudo:session): session closed for user root 2024-04-15T21:31:39.582 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:39 smithi083 ceph-mon[24790]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-15T21:31:39.582 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-04-15T21:31:39.582 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi139"}]: dispatch 2024-04-15T21:31:39.582 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:39 smithi083 ceph-mon[24790]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-15T21:31:39.582 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-04-15T21:31:39.582 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi139"}]: dispatch 2024-04-15T21:31:39.582 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:39 smithi083 ceph-mon[24790]: pgmap v75: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-15T21:31:39.583 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:39 smithi083 ceph-mon[24790]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-15T21:31:39.583 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:39 smithi083 ceph-mon[24790]: mgrmap e19: smithi083.kxqarn(active, since 2m), standbys: smithi139.bqqykn 2024-04-15T21:31:39.583 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:39.583 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:39.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:39 smithi139 ceph-mon[26919]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-15T21:31:39.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-04-15T21:31:39.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi139"}]: dispatch 2024-04-15T21:31:39.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:39 smithi139 ceph-mon[26919]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-15T21:31:39.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-04-15T21:31:39.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mon metadata", "id": "smithi139"}]: dispatch 2024-04-15T21:31:39.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:39 smithi139 ceph-mon[26919]: pgmap v75: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-15T21:31:39.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:39 smithi139 ceph-mon[26919]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-15T21:31:39.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:39 smithi139 ceph-mon[26919]: mgrmap e19: smithi083.kxqarn(active, since 2m), standbys: smithi139.bqqykn 2024-04-15T21:31:39.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:39.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:40.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:40 smithi083 ceph-mon[24790]: from='client.14294 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:31:40.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:40 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-15T21:31:40.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:40 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-15T21:31:40.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:40 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:31:40.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:40 smithi139 ceph-mon[26919]: from='client.14294 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:31:40.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:40 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-15T21:31:40.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:40 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-15T21:31:40.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:40 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:31:41.362 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:41 smithi139 ceph-mon[26919]: pgmap v76: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-15T21:31:41.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:41 smithi083 ceph-mon[24790]: pgmap v76: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-15T21:31:43.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:43 smithi083 ceph-mon[24790]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 218 KiB/s wr, 14 op/s 2024-04-15T21:31:43.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:43 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:43.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:43 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:31:43.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:43 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:31:43.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:43 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:31:43.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:43 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:43.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:43 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:43.827 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:43 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:31:43.827 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:43 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:31:43.827 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:43 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:31:43.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:43 smithi139 ceph-mon[26919]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 218 KiB/s wr, 14 op/s 2024-04-15T21:31:43.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:43 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:43.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:43 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:31:43.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:43 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:31:43.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:43 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:31:43.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:43 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:43.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:43 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:43.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:43 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:31:43.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:43 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:31:43.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:43 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:31:44.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:44 smithi083 ceph-mon[24790]: Adjusting osd_memory_target on smithi083 to 4288M 2024-04-15T21:31:44.705 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:44 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1506027096' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8ea0f736-4b9b-4015-a82b-dcf9278d51b4"}]: dispatch 2024-04-15T21:31:44.705 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:44 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1506027096' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8ea0f736-4b9b-4015-a82b-dcf9278d51b4"}]': finished 2024-04-15T21:31:44.705 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:44 smithi083 ceph-mon[24790]: osdmap e23: 4 total, 3 up, 4 in 2024-04-15T21:31:44.705 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:44 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-15T21:31:44.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:44 smithi139 ceph-mon[26919]: Adjusting osd_memory_target on smithi083 to 4288M 2024-04-15T21:31:44.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:44 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/1506027096' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8ea0f736-4b9b-4015-a82b-dcf9278d51b4"}]: dispatch 2024-04-15T21:31:44.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:44 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/1506027096' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8ea0f736-4b9b-4015-a82b-dcf9278d51b4"}]': finished 2024-04-15T21:31:44.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:44 smithi139 ceph-mon[26919]: osdmap e23: 4 total, 3 up, 4 in 2024-04-15T21:31:44.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:44 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-15T21:31:45.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:45 smithi083 ceph-mon[24790]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.4 KiB/s rd, 163 KiB/s wr, 11 op/s 2024-04-15T21:31:45.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:45 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1229731753' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-15T21:31:45.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:45 smithi139 ceph-mon[26919]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.4 KiB/s rd, 163 KiB/s wr, 11 op/s 2024-04-15T21:31:45.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:45 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/1229731753' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-15T21:31:47.668 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:47 smithi083 ceph-mon[24790]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.8 KiB/s rd, 135 KiB/s wr, 9 op/s 2024-04-15T21:31:47.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:47 smithi139 ceph-mon[26919]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.8 KiB/s rd, 135 KiB/s wr, 9 op/s 2024-04-15T21:31:49.105 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:48 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:49.105 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:48 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:49.105 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:48 smithi083 ceph-mon[24790]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-04-15T21:31:49.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:48 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:49.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:48 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:49.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:48 smithi139 ceph-mon[26919]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-04-15T21:31:50.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-15T21:31:50.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:50 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:31:50.706 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:50 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-15T21:31:50.706 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:50 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:31:51.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:51 smithi083 ceph-mon[24790]: Deploying daemon osd.3 on smithi083 2024-04-15T21:31:51.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:51 smithi083 ceph-mon[24790]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-04-15T21:31:51.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:51 smithi139 ceph-mon[26919]: Deploying daemon osd.3 on smithi083 2024-04-15T21:31:51.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:51 smithi139 ceph-mon[26919]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-04-15T21:31:53.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:53 smithi083 ceph-mon[24790]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-15T21:31:53.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:53 smithi139 ceph-mon[26919]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-15T21:31:55.596 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:55 smithi083 ceph-mon[24790]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-15T21:31:55.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:55 smithi139 ceph-mon[26919]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-15T21:31:56.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:56 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:56.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:56 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:31:56.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:56 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:31:56.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:56 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:31:56.882 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:56 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:56.882 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:56 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:31:56.882 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:56 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:31:56.882 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:56 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:31:57.808 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:57 smithi083 ceph-mon[24790]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-15T21:31:57.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:57 smithi139 ceph-mon[26919]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-15T21:31:59.063 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 3 on host 'smithi083' 2024-04-15T21:31:59.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:59 smithi083 ceph-mon[24790]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-15T21:31:59.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:59 smithi083 ceph-mon[24790]: from='osd.3 [v2:172.21.15.83:6826/2705135586,v1:172.21.15.83:6827/2705135586]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-15T21:31:59.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:31:59 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:31:59.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:59 smithi139 ceph-mon[26919]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-15T21:31:59.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:59 smithi139 ceph-mon[26919]: from='osd.3 [v2:172.21.15.83:6826/2705135586,v1:172.21.15.83:6827/2705135586]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-15T21:31:59.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:31:59 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:00.045 DEBUG:teuthology.orchestra.run.smithi083:osd.3> sudo journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.3.service 2024-04-15T21:32:00.047 INFO:tasks.cephadm:Deploying osd.4 on smithi139 with /dev/vg_nvme/lv_4... 2024-04-15T21:32:00.047 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-15T21:32:01.000 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:00 smithi083 ceph-mon[24790]: from='osd.3 [v2:172.21.15.83:6826/2705135586,v1:172.21.15.83:6827/2705135586]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-15T21:32:01.001 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:00 smithi083 ceph-mon[24790]: osdmap e24: 4 total, 3 up, 4 in 2024-04-15T21:32:01.001 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:00 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-15T21:32:01.001 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:00 smithi083 ceph-mon[24790]: from='osd.3 [v2:172.21.15.83:6826/2705135586,v1:172.21.15.83:6827/2705135586]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-15T21:32:01.001 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:32:00 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-3[45745]: 2024-04-15T21:32:00.620+0000 7f8a0a75b700 -1 osd.3 0 waiting for initial osdmap 2024-04-15T21:32:01.001 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:32:00 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-3[45745]: 2024-04-15T21:32:00.626+0000 7f8a030ee700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-15T21:32:01.097 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:00 smithi139 ceph-mon[26919]: from='osd.3 [v2:172.21.15.83:6826/2705135586,v1:172.21.15.83:6827/2705135586]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-15T21:32:01.097 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:00 smithi139 ceph-mon[26919]: osdmap e24: 4 total, 3 up, 4 in 2024-04-15T21:32:01.097 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:00 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-15T21:32:01.097 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:00 smithi139 ceph-mon[26919]: from='osd.3 [v2:172.21.15.83:6826/2705135586,v1:172.21.15.83:6827/2705135586]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-15T21:32:01.738 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:01 smithi139 ceph-mon[26919]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-15T21:32:01.738 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:01 smithi139 ceph-mon[26919]: from='osd.3 [v2:172.21.15.83:6826/2705135586,v1:172.21.15.83:6827/2705135586]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-15T21:32:01.738 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:01 smithi139 ceph-mon[26919]: osdmap e25: 4 total, 3 up, 4 in 2024-04-15T21:32:01.738 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:01 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-15T21:32:01.738 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:01 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-15T21:32:01.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:01 smithi083 ceph-mon[24790]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-15T21:32:01.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:01 smithi083 ceph-mon[24790]: from='osd.3 [v2:172.21.15.83:6826/2705135586,v1:172.21.15.83:6827/2705135586]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-15T21:32:01.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:01 smithi083 ceph-mon[24790]: osdmap e25: 4 total, 3 up, 4 in 2024-04-15T21:32:01.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:01 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-15T21:32:01.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:01 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-15T21:32:02.235 INFO:teuthology.orchestra.run.smithi139.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-15T21:32:02.236 INFO:teuthology.orchestra.run.smithi139.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-15T21:32:02.236 INFO:teuthology.orchestra.run.smithi139.stderr: stderr: 10+0 records in 2024-04-15T21:32:02.236 INFO:teuthology.orchestra.run.smithi139.stderr:10+0 records out 2024-04-15T21:32:02.236 INFO:teuthology.orchestra.run.smithi139.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0137694 s, 762 MB/s 2024-04-15T21:32:02.236 INFO:teuthology.orchestra.run.smithi139.stderr:--> Zapping successful for: 2024-04-15T21:32:02.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:02 smithi139 ceph-mon[26919]: purged_snaps scrub starts 2024-04-15T21:32:02.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:02 smithi139 ceph-mon[26919]: purged_snaps scrub ok 2024-04-15T21:32:02.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:02 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-15T21:32:02.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:02 smithi139 ceph-mon[26919]: osd.3 [v2:172.21.15.83:6826/2705135586,v1:172.21.15.83:6827/2705135586] boot 2024-04-15T21:32:02.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:02 smithi139 ceph-mon[26919]: osdmap e26: 4 total, 4 up, 4 in 2024-04-15T21:32:02.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:02 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-15T21:32:02.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:02 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:02.909 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:02 smithi083 ceph-mon[24790]: purged_snaps scrub starts 2024-04-15T21:32:02.909 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:02 smithi083 ceph-mon[24790]: purged_snaps scrub ok 2024-04-15T21:32:02.909 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:02 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-15T21:32:02.909 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:02 smithi083 ceph-mon[24790]: osd.3 [v2:172.21.15.83:6826/2705135586,v1:172.21.15.83:6827/2705135586] boot 2024-04-15T21:32:02.909 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:02 smithi083 ceph-mon[24790]: osdmap e26: 4 total, 4 up, 4 in 2024-04-15T21:32:02.909 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:02 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-15T21:32:02.909 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:02 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:02.936 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph orch daemon add osd smithi139:vg_nvme/lv_4 2024-04-15T21:32:03.790 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:03 smithi083 ceph-mon[24790]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-15T21:32:03.790 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:03 smithi083 ceph-mon[24790]: osdmap e27: 4 total, 4 up, 4 in 2024-04-15T21:32:03.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:03 smithi139 ceph-mon[26919]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-15T21:32:03.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:03 smithi139 ceph-mon[26919]: osdmap e27: 4 total, 4 up, 4 in 2024-04-15T21:32:05.826 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:05 smithi139 ceph-mon[26919]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:05.826 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:05 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-15T21:32:05.826 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:05 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-15T21:32:05.826 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:05 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:05.826 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:05 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:05.826 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:05 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:32:05.826 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:05 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:32:05.827 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:05 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:32:05.827 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:05 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:32:05.827 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:05 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:05.827 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:05 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:05.827 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:05 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:32:05.827 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:05 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:05.827 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:05 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:32:06.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:05 smithi083 ceph-mon[24790]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:06.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:05 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-15T21:32:06.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:05 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-15T21:32:06.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:05 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:06.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:05 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:06.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:05 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:32:06.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:05 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:32:06.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:05 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:32:06.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:05 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:32:06.102 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:05 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:06.102 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:05 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:06.102 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:05 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:32:06.102 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:05 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:06.102 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:05 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:32:06.705 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:06 smithi083 ceph-mon[24790]: from='client.24151 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:32:06.706 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:06 smithi083 ceph-mon[24790]: Detected new or changed devices on smithi083 2024-04-15T21:32:06.706 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:06 smithi083 ceph-mon[24790]: Adjusting osd_memory_target on smithi083 to 3216M 2024-04-15T21:32:06.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:06 smithi139 ceph-mon[26919]: from='client.24151 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:32:06.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:06 smithi139 ceph-mon[26919]: Detected new or changed devices on smithi083 2024-04-15T21:32:06.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:06 smithi139 ceph-mon[26919]: Adjusting osd_memory_target on smithi083 to 3216M 2024-04-15T21:32:08.043 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:07 smithi083 ceph-mon[24790]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:08.043 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:07 smithi083 ceph-mon[24790]: from='client.? 172.21.15.139:0/3680273711' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9a9d5120-0512-44aa-b6f4-8e70170e861f"}]: dispatch 2024-04-15T21:32:08.043 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:07 smithi083 ceph-mon[24790]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9a9d5120-0512-44aa-b6f4-8e70170e861f"}]: dispatch 2024-04-15T21:32:08.043 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:07 smithi083 ceph-mon[24790]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9a9d5120-0512-44aa-b6f4-8e70170e861f"}]': finished 2024-04-15T21:32:08.043 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:07 smithi083 ceph-mon[24790]: osdmap e28: 5 total, 4 up, 5 in 2024-04-15T21:32:08.043 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:07 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-15T21:32:08.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:07 smithi139 ceph-mon[26919]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:08.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:07 smithi139 ceph-mon[26919]: from='client.? 172.21.15.139:0/3680273711' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9a9d5120-0512-44aa-b6f4-8e70170e861f"}]: dispatch 2024-04-15T21:32:08.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:07 smithi139 ceph-mon[26919]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9a9d5120-0512-44aa-b6f4-8e70170e861f"}]: dispatch 2024-04-15T21:32:08.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:07 smithi139 ceph-mon[26919]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9a9d5120-0512-44aa-b6f4-8e70170e861f"}]': finished 2024-04-15T21:32:08.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:07 smithi139 ceph-mon[26919]: osdmap e28: 5 total, 4 up, 5 in 2024-04-15T21:32:08.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:07 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-15T21:32:08.977 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:08 smithi083 ceph-mon[24790]: from='client.? 172.21.15.139:0/3491872394' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-15T21:32:09.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:08 smithi139 ceph-mon[26919]: from='client.? 172.21.15.139:0/3491872394' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-15T21:32:09.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:09 smithi083 ceph-mon[24790]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:09.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:09 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:09.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:09 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:10.021 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:09 smithi139 ceph-mon[26919]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:10.021 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:09 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:10.021 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:09 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:11.791 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:11 smithi139 ceph-mon[26919]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:12.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:11 smithi083 ceph-mon[24790]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:13.924 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:13 smithi139 ceph-mon[26919]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:13.925 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:13 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-15T21:32:13.925 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:13 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:14.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:13 smithi083 ceph-mon[24790]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:14.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:13 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-15T21:32:14.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:13 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:14.912 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:14 smithi139 ceph-mon[26919]: Deploying daemon osd.4 on smithi139 2024-04-15T21:32:15.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:14 smithi083 ceph-mon[24790]: Deploying daemon osd.4 on smithi139 2024-04-15T21:32:15.943 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:15 smithi139 ceph-mon[26919]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:16.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:15 smithi083 ceph-mon[24790]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:17.834 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:17 smithi139 ceph-mon[26919]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:18.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:17 smithi083 ceph-mon[24790]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:19.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:19.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:32:19.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:19.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:32:19.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:19 smithi139 ceph-mon[26919]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:19.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:19.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:32:19.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:19.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:32:19.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:19 smithi083 ceph-mon[24790]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:21.260 INFO:teuthology.orchestra.run.smithi139.stdout:Created osd(s) 4 on host 'smithi139' 2024-04-15T21:32:21.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:21 smithi083 ceph-mon[24790]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:21.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:21 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:21.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:21 smithi083 ceph-mon[24790]: from='osd.4 [v2:172.21.15.139:6800/962351193,v1:172.21.15.139:6801/962351193]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-15T21:32:21.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:21 smithi083 ceph-mon[24790]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-15T21:32:21.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:21 smithi139 ceph-mon[26919]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:21.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:21 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:21.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:21 smithi139 ceph-mon[26919]: from='osd.4 [v2:172.21.15.139:6800/962351193,v1:172.21.15.139:6801/962351193]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-15T21:32:21.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:21 smithi139 ceph-mon[26919]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-15T21:32:22.287 DEBUG:teuthology.orchestra.run.smithi139:osd.4> sudo journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.4.service 2024-04-15T21:32:22.290 INFO:tasks.cephadm:Deploying osd.5 on smithi139 with /dev/vg_nvme/lv_3... 2024-04-15T21:32:22.290 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-15T21:32:23.529 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:23 smithi139 ceph-mon[26919]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-15T21:32:23.529 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:23 smithi139 ceph-mon[26919]: osdmap e29: 5 total, 4 up, 5 in 2024-04-15T21:32:23.529 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:23 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-15T21:32:23.529 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:23 smithi139 ceph-mon[26919]: from='osd.4 [v2:172.21.15.139:6800/962351193,v1:172.21.15.139:6801/962351193]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2024-04-15T21:32:23.530 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:23 smithi139 ceph-mon[26919]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2024-04-15T21:32:23.530 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:23 smithi139 ceph-mon[26919]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:23.530 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:23 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:23.530 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:32:23 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4[31176]: 2024-04-15T21:32:23.274+0000 7f2532759700 -1 osd.4 0 waiting for initial osdmap 2024-04-15T21:32:23.531 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:32:23 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4[31176]: 2024-04-15T21:32:23.278+0000 7f252b0ec700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-15T21:32:23.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:23 smithi083 ceph-mon[24790]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-15T21:32:23.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:23 smithi083 ceph-mon[24790]: osdmap e29: 5 total, 4 up, 5 in 2024-04-15T21:32:23.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:23 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-15T21:32:23.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:23 smithi083 ceph-mon[24790]: from='osd.4 [v2:172.21.15.139:6800/962351193,v1:172.21.15.139:6801/962351193]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2024-04-15T21:32:23.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:23 smithi083 ceph-mon[24790]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2024-04-15T21:32:23.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:23 smithi083 ceph-mon[24790]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-15T21:32:23.602 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:23 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:24.382 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:24 smithi139 ceph-mon[26919]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2024-04-15T21:32:24.382 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:24 smithi139 ceph-mon[26919]: osdmap e30: 5 total, 4 up, 5 in 2024-04-15T21:32:24.382 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:24 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-15T21:32:24.383 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:24 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-15T21:32:24.383 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:24 smithi139 ceph-mon[26919]: osd.4 [v2:172.21.15.139:6800/962351193,v1:172.21.15.139:6801/962351193] boot 2024-04-15T21:32:24.383 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:24 smithi139 ceph-mon[26919]: osdmap e31: 5 total, 5 up, 5 in 2024-04-15T21:32:24.383 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:24 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-15T21:32:24.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:24 smithi083 ceph-mon[24790]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2024-04-15T21:32:24.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:24 smithi083 ceph-mon[24790]: osdmap e30: 5 total, 4 up, 5 in 2024-04-15T21:32:24.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:24 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-15T21:32:24.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:24 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-15T21:32:24.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:24 smithi083 ceph-mon[24790]: osd.4 [v2:172.21.15.139:6800/962351193,v1:172.21.15.139:6801/962351193] boot 2024-04-15T21:32:24.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:24 smithi083 ceph-mon[24790]: osdmap e31: 5 total, 5 up, 5 in 2024-04-15T21:32:24.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:24 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-15T21:32:24.867 INFO:teuthology.orchestra.run.smithi139.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-15T21:32:24.867 INFO:teuthology.orchestra.run.smithi139.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-15T21:32:24.867 INFO:teuthology.orchestra.run.smithi139.stderr: stderr: 10+0 records in 2024-04-15T21:32:24.868 INFO:teuthology.orchestra.run.smithi139.stderr:10+0 records out 2024-04-15T21:32:24.868 INFO:teuthology.orchestra.run.smithi139.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0125323 s, 837 MB/s 2024-04-15T21:32:24.868 INFO:teuthology.orchestra.run.smithi139.stderr:--> Zapping successful for: 2024-04-15T21:32:25.577 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:25 smithi139 ceph-mon[26919]: purged_snaps scrub starts 2024-04-15T21:32:25.577 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:25 smithi139 ceph-mon[26919]: purged_snaps scrub ok 2024-04-15T21:32:25.577 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:25 smithi139 ceph-mon[26919]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-15T21:32:25.577 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:25 smithi139 ceph-mon[26919]: osdmap e32: 5 total, 5 up, 5 in 2024-04-15T21:32:25.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:25 smithi083 ceph-mon[24790]: purged_snaps scrub starts 2024-04-15T21:32:25.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:25 smithi083 ceph-mon[24790]: purged_snaps scrub ok 2024-04-15T21:32:25.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:25 smithi083 ceph-mon[24790]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-15T21:32:25.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:25 smithi083 ceph-mon[24790]: osdmap e32: 5 total, 5 up, 5 in 2024-04-15T21:32:26.248 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph orch daemon add osd smithi139:vg_nvme/lv_3 2024-04-15T21:32:27.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:27 smithi083 ceph-mon[24790]: osdmap e33: 5 total, 5 up, 5 in 2024-04-15T21:32:27.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:27 smithi083 ceph-mon[24790]: pgmap v110: 1 pgs: 1 active+recovering+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-15T21:32:27.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:27 smithi139 ceph-mon[26919]: osdmap e33: 5 total, 5 up, 5 in 2024-04-15T21:32:27.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:27 smithi139 ceph-mon[26919]: pgmap v110: 1 pgs: 1 active+recovering+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-15T21:32:28.362 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:28 smithi139 ceph-mon[26919]: osdmap e34: 5 total, 5 up, 5 in 2024-04-15T21:32:28.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:28 smithi083 ceph-mon[24790]: osdmap e34: 5 total, 5 up, 5 in 2024-04-15T21:32:29.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:29 smithi083 ceph-mon[24790]: osdmap e35: 5 total, 5 up, 5 in 2024-04-15T21:32:29.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:29 smithi083 ceph-mon[24790]: pgmap v113: 1 pgs: 1 active+recovering+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-15T21:32:29.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:32:29.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:32:29.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-15T21:32:29.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-15T21:32:29.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:29.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:29.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:32:29.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:29.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:29.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:32:29.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:29.602 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:32:29.602 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:29.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:29 smithi139 ceph-mon[26919]: osdmap e35: 5 total, 5 up, 5 in 2024-04-15T21:32:29.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:29 smithi139 ceph-mon[26919]: pgmap v113: 1 pgs: 1 active+recovering+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-15T21:32:29.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:29 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:32:29.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:29 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:32:29.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:29 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-15T21:32:29.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:29 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-15T21:32:29.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:29 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:29.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:29 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:29.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:29 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:32:29.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:29 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:29.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:29 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:29.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:29 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:32:29.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:29 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:29.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:29 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:32:29.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:29 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:30.362 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:30 smithi139 ceph-mon[26919]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:32:30.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:30 smithi139 ceph-mon[26919]: Detected new or changed devices on smithi139 2024-04-15T21:32:30.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:30 smithi139 ceph-mon[26919]: Adjusting osd_memory_target on smithi139 to 15936M 2024-04-15T21:32:30.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:30 smithi083 ceph-mon[24790]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:32:30.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:30 smithi083 ceph-mon[24790]: Detected new or changed devices on smithi139 2024-04-15T21:32:30.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:30 smithi083 ceph-mon[24790]: Adjusting osd_memory_target on smithi139 to 15936M 2024-04-15T21:32:31.350 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:31 smithi083 ceph-mon[24790]: pgmap v114: 1 pgs: 1 active+recovering+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-04-15T21:32:31.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:31 smithi083 ceph-mon[24790]: from='client.? 172.21.15.139:0/1377871727' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fd246cf7-ef0b-4cca-85ce-3dfbc73d75a2"}]: dispatch 2024-04-15T21:32:31.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:31 smithi083 ceph-mon[24790]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fd246cf7-ef0b-4cca-85ce-3dfbc73d75a2"}]: dispatch 2024-04-15T21:32:31.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:31 smithi083 ceph-mon[24790]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fd246cf7-ef0b-4cca-85ce-3dfbc73d75a2"}]': finished 2024-04-15T21:32:31.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:31 smithi083 ceph-mon[24790]: osdmap e36: 6 total, 5 up, 6 in 2024-04-15T21:32:31.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:31 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-15T21:32:31.362 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:31 smithi139 ceph-mon[26919]: pgmap v114: 1 pgs: 1 active+recovering+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-04-15T21:32:31.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:31 smithi139 ceph-mon[26919]: from='client.? 172.21.15.139:0/1377871727' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fd246cf7-ef0b-4cca-85ce-3dfbc73d75a2"}]: dispatch 2024-04-15T21:32:31.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:31 smithi139 ceph-mon[26919]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fd246cf7-ef0b-4cca-85ce-3dfbc73d75a2"}]: dispatch 2024-04-15T21:32:31.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:31 smithi139 ceph-mon[26919]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fd246cf7-ef0b-4cca-85ce-3dfbc73d75a2"}]': finished 2024-04-15T21:32:31.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:31 smithi139 ceph-mon[26919]: osdmap e36: 6 total, 5 up, 6 in 2024-04-15T21:32:31.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:31 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-15T21:32:32.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:32 smithi083 ceph-mon[24790]: from='client.? 172.21.15.139:0/280032263' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-15T21:32:32.612 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:32 smithi139 ceph-mon[26919]: from='client.? 172.21.15.139:0/280032263' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-15T21:32:33.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:33 smithi083 ceph-mon[24790]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-15T21:32:33.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:33 smithi139 ceph-mon[26919]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-15T21:32:35.841 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:35 smithi139 ceph-mon[26919]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-15T21:32:35.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:35 smithi083 ceph-mon[24790]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-15T21:32:37.705 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:37 smithi139 ceph-mon[26919]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-15T21:32:37.705 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:37 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-15T21:32:37.705 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:37 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:37.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:37 smithi083 ceph-mon[24790]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-15T21:32:37.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:37 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-15T21:32:37.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:37 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:38.568 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:38 smithi139 ceph-mon[26919]: Deploying daemon osd.5 on smithi139 2024-04-15T21:32:38.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:38 smithi083 ceph-mon[24790]: Deploying daemon osd.5 on smithi139 2024-04-15T21:32:39.844 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:39 smithi139 ceph-mon[26919]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-15T21:32:39.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:39 smithi083 ceph-mon[24790]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-15T21:32:41.743 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:41 smithi139 ceph-mon[26919]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-15T21:32:41.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:41 smithi083 ceph-mon[24790]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-15T21:32:43.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:43 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:43.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:43 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:32:43.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:43 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:43.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:43 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:32:43.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:43 smithi083 ceph-mon[24790]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-15T21:32:43.602 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:43 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:43.603 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:43 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:32:43.603 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:43 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:43.603 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:43 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:32:43.603 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:43 smithi139 ceph-mon[26919]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-15T21:32:45.689 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:45 smithi139 ceph-mon[26919]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-15T21:32:45.689 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:45 smithi139 ceph-mon[26919]: from='osd.5 [v2:172.21.15.139:6808/2653213923,v1:172.21.15.139:6809/2653213923]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-15T21:32:45.689 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:45 smithi139 ceph-mon[26919]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-15T21:32:45.722 INFO:teuthology.orchestra.run.smithi139.stdout:Created osd(s) 5 on host 'smithi139' 2024-04-15T21:32:45.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:45 smithi083 ceph-mon[24790]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-15T21:32:45.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:45 smithi083 ceph-mon[24790]: from='osd.5 [v2:172.21.15.139:6808/2653213923,v1:172.21.15.139:6809/2653213923]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-15T21:32:45.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:45 smithi083 ceph-mon[24790]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-15T21:32:46.349 DEBUG:teuthology.orchestra.run.smithi139:osd.5> sudo journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.5.service 2024-04-15T21:32:46.351 INFO:tasks.cephadm:Deploying osd.6 on smithi139 with /dev/vg_nvme/lv_2... 2024-04-15T21:32:46.351 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-15T21:32:46.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:46 smithi083 ceph-mon[24790]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-15T21:32:46.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:46 smithi083 ceph-mon[24790]: osdmap e37: 6 total, 5 up, 6 in 2024-04-15T21:32:46.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:46 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-15T21:32:46.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:46 smithi083 ceph-mon[24790]: from='osd.5 [v2:172.21.15.139:6808/2653213923,v1:172.21.15.139:6809/2653213923]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2024-04-15T21:32:46.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:46 smithi083 ceph-mon[24790]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2024-04-15T21:32:46.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:46 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:46.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:46 smithi139 ceph-mon[26919]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-15T21:32:46.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:46 smithi139 ceph-mon[26919]: osdmap e37: 6 total, 5 up, 6 in 2024-04-15T21:32:46.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:46 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-15T21:32:46.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:46 smithi139 ceph-mon[26919]: from='osd.5 [v2:172.21.15.139:6808/2653213923,v1:172.21.15.139:6809/2653213923]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2024-04-15T21:32:46.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:46 smithi139 ceph-mon[26919]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2024-04-15T21:32:46.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:46 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:46.863 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:32:46 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:32:46.582+0000 7fb91694d700 -1 osd.5 0 waiting for initial osdmap 2024-04-15T21:32:46.863 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:32:46 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:32:46.587+0000 7fb9132e8700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-15T21:32:47.781 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:47 smithi139 ceph-mon[26919]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-15T21:32:47.781 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:47 smithi139 ceph-mon[26919]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2024-04-15T21:32:47.781 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:47 smithi139 ceph-mon[26919]: osdmap e38: 6 total, 5 up, 6 in 2024-04-15T21:32:47.781 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:47 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-15T21:32:47.781 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:47 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-15T21:32:47.781 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:47 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:47.781 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:47 smithi139 ceph-mon[26919]: osd.5 [v2:172.21.15.139:6808/2653213923,v1:172.21.15.139:6809/2653213923] boot 2024-04-15T21:32:47.781 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:47 smithi139 ceph-mon[26919]: osdmap e39: 6 total, 6 up, 6 in 2024-04-15T21:32:47.782 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:47 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-15T21:32:47.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:47 smithi083 ceph-mon[24790]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-15T21:32:47.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:47 smithi083 ceph-mon[24790]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2024-04-15T21:32:47.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:47 smithi083 ceph-mon[24790]: osdmap e38: 6 total, 5 up, 6 in 2024-04-15T21:32:47.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:47 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-15T21:32:47.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:47 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-15T21:32:47.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:47 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:47.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:47 smithi083 ceph-mon[24790]: osd.5 [v2:172.21.15.139:6808/2653213923,v1:172.21.15.139:6809/2653213923] boot 2024-04-15T21:32:47.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:47 smithi083 ceph-mon[24790]: osdmap e39: 6 total, 6 up, 6 in 2024-04-15T21:32:47.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:47 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-15T21:32:48.609 INFO:teuthology.orchestra.run.smithi139.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-15T21:32:48.610 INFO:teuthology.orchestra.run.smithi139.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-15T21:32:48.610 INFO:teuthology.orchestra.run.smithi139.stderr: stderr: 10+0 records in 2024-04-15T21:32:48.610 INFO:teuthology.orchestra.run.smithi139.stderr:10+0 records out 2024-04-15T21:32:48.610 INFO:teuthology.orchestra.run.smithi139.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0123622 s, 848 MB/s 2024-04-15T21:32:48.610 INFO:teuthology.orchestra.run.smithi139.stderr:--> Zapping successful for: 2024-04-15T21:32:48.690 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:48 smithi139 ceph-mon[26919]: purged_snaps scrub starts 2024-04-15T21:32:48.691 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:48 smithi139 ceph-mon[26919]: purged_snaps scrub ok 2024-04-15T21:32:48.691 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:48 smithi139 ceph-mon[26919]: osdmap e40: 6 total, 6 up, 6 in 2024-04-15T21:32:48.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:48 smithi083 ceph-mon[24790]: purged_snaps scrub starts 2024-04-15T21:32:48.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:48 smithi083 ceph-mon[24790]: purged_snaps scrub ok 2024-04-15T21:32:48.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:48 smithi083 ceph-mon[24790]: osdmap e40: 6 total, 6 up, 6 in 2024-04-15T21:32:49.772 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:49 smithi139 ceph-mon[26919]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:32:49.772 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:49 smithi139 ceph-mon[26919]: osdmap e41: 6 total, 6 up, 6 in 2024-04-15T21:32:49.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:49 smithi083 ceph-mon[24790]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:32:49.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:49 smithi083 ceph-mon[24790]: osdmap e41: 6 total, 6 up, 6 in 2024-04-15T21:32:50.048 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph orch daemon add osd smithi139:vg_nvme/lv_2 2024-04-15T21:32:51.809 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:51 smithi139 ceph-mon[26919]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:32:51.809 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:51 smithi139 ceph-mon[26919]: osdmap e42: 6 total, 6 up, 6 in 2024-04-15T21:32:52.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:51 smithi083 ceph-mon[24790]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:32:52.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:51 smithi083 ceph-mon[24790]: osdmap e42: 6 total, 6 up, 6 in 2024-04-15T21:32:52.748 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:52 smithi139 ceph-mon[26919]: osdmap e43: 6 total, 6 up, 6 in 2024-04-15T21:32:52.748 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:52 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:52.748 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:52 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:32:52.748 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:52 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:32:52.748 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:52 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:52.748 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:52 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:52.748 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:52 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:32:52.748 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:52 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:52.749 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:52 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:32:52.749 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:52 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:53.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:52 smithi083 ceph-mon[24790]: osdmap e43: 6 total, 6 up, 6 in 2024-04-15T21:32:53.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:52 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:53.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:52 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:32:53.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:52 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:32:53.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:52 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:53.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:52 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:53.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:52 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:32:53.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:52 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:53.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:52 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:32:53.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:52 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:32:53.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:53 smithi139 ceph-mon[26919]: Detected new or changed devices on smithi139 2024-04-15T21:32:53.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:53 smithi139 ceph-mon[26919]: Adjusting osd_memory_target on smithi139 to 7968M 2024-04-15T21:32:53.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:53 smithi139 ceph-mon[26919]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:32:53.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:53 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-15T21:32:53.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:53 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-15T21:32:53.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:53 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:54.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:53 smithi083 ceph-mon[24790]: Detected new or changed devices on smithi139 2024-04-15T21:32:54.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:53 smithi083 ceph-mon[24790]: Adjusting osd_memory_target on smithi139 to 7968M 2024-04-15T21:32:54.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:53 smithi083 ceph-mon[24790]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:32:54.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:53 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-15T21:32:54.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:53 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-15T21:32:54.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:53 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:32:54.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:54 smithi139 ceph-mon[26919]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:32:55.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:54 smithi083 ceph-mon[24790]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:32:55.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:55 smithi139 ceph-mon[26919]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:32:55.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:55 smithi139 ceph-mon[26919]: from='client.? 172.21.15.139:0/1843480498' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fed409f9-7882-4a83-8667-79afe110ad61"}]: dispatch 2024-04-15T21:32:55.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:55 smithi139 ceph-mon[26919]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fed409f9-7882-4a83-8667-79afe110ad61"}]: dispatch 2024-04-15T21:32:55.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:55 smithi139 ceph-mon[26919]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fed409f9-7882-4a83-8667-79afe110ad61"}]': finished 2024-04-15T21:32:55.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:55 smithi139 ceph-mon[26919]: osdmap e44: 7 total, 6 up, 7 in 2024-04-15T21:32:55.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:55 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-15T21:32:56.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:55 smithi083 ceph-mon[24790]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:32:56.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:55 smithi083 ceph-mon[24790]: from='client.? 172.21.15.139:0/1843480498' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fed409f9-7882-4a83-8667-79afe110ad61"}]: dispatch 2024-04-15T21:32:56.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:55 smithi083 ceph-mon[24790]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fed409f9-7882-4a83-8667-79afe110ad61"}]: dispatch 2024-04-15T21:32:56.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:55 smithi083 ceph-mon[24790]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fed409f9-7882-4a83-8667-79afe110ad61"}]': finished 2024-04-15T21:32:56.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:55 smithi083 ceph-mon[24790]: osdmap e44: 7 total, 6 up, 7 in 2024-04-15T21:32:56.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:55 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-15T21:32:57.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:56 smithi083 ceph-mon[24790]: from='client.? 172.21.15.139:0/2717449664' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-15T21:32:57.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:56 smithi139 ceph-mon[26919]: from='client.? 172.21.15.139:0/2717449664' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-15T21:32:58.006 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:57 smithi139 ceph-mon[26919]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:32:58.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:57 smithi083 ceph-mon[24790]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:32:59.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:32:59 smithi083 ceph-mon[24790]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:32:59.976 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:32:59 smithi139 ceph-mon[26919]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:33:01.861 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:01 smithi139 ceph-mon[26919]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:33:01.861 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:01 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-15T21:33:01.861 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:01 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:33:02.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:01 smithi083 ceph-mon[24790]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:33:02.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:01 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-15T21:33:02.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:01 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:33:02.822 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:02 smithi139 ceph-mon[26919]: Deploying daemon osd.6 on smithi139 2024-04-15T21:33:03.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:02 smithi083 ceph-mon[24790]: Deploying daemon osd.6 on smithi139 2024-04-15T21:33:03.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:03 smithi139 ceph-mon[26919]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:33:04.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:03 smithi083 ceph-mon[24790]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:33:06.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:05 smithi083 ceph-mon[24790]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:33:06.103 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:05 smithi139 ceph-mon[26919]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:33:08.029 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:07 smithi139 ceph-mon[26919]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:33:08.029 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:07 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:08.029 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:07 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:33:08.029 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:07 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:33:08.030 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:07 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:33:08.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:07 smithi083 ceph-mon[24790]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:33:08.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:07 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:08.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:07 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:33:08.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:07 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:33:08.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:07 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:33:09.738 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:09 smithi139 ceph-mon[26919]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:33:09.769 INFO:teuthology.orchestra.run.smithi139.stdout:Created osd(s) 6 on host 'smithi139' 2024-04-15T21:33:10.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:09 smithi083 ceph-mon[24790]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:33:10.390 DEBUG:teuthology.orchestra.run.smithi139:osd.6> sudo journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.6.service 2024-04-15T21:33:10.393 INFO:tasks.cephadm:Deploying osd.7 on smithi139 with /dev/vg_nvme/lv_1... 2024-04-15T21:33:10.393 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-15T21:33:11.061 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:10 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:11.061 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:10 smithi139 ceph-mon[26919]: from='osd.6 [v2:172.21.15.139:6816/121728892,v1:172.21.15.139:6817/121728892]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-15T21:33:11.062 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:10 smithi139 ceph-mon[26919]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-15T21:33:11.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:10 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:11.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:10 smithi083 ceph-mon[24790]: from='osd.6 [v2:172.21.15.139:6816/121728892,v1:172.21.15.139:6817/121728892]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-15T21:33:11.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:10 smithi083 ceph-mon[24790]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-15T21:33:11.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:11 smithi139 ceph-mon[26919]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:33:11.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:11 smithi139 ceph-mon[26919]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-15T21:33:11.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:11 smithi139 ceph-mon[26919]: osdmap e45: 7 total, 6 up, 7 in 2024-04-15T21:33:11.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:11 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-15T21:33:11.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:11 smithi139 ceph-mon[26919]: from='osd.6 [v2:172.21.15.139:6816/121728892,v1:172.21.15.139:6817/121728892]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2024-04-15T21:33:11.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:11 smithi139 ceph-mon[26919]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2024-04-15T21:33:11.864 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:33:11 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:33:11.783+0000 7f78f35af700 -1 osd.6 0 waiting for initial osdmap 2024-04-15T21:33:11.864 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:33:11 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:33:11.789+0000 7f78ee747700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-15T21:33:12.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:11 smithi083 ceph-mon[24790]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:33:12.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:11 smithi083 ceph-mon[24790]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-15T21:33:12.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:11 smithi083 ceph-mon[24790]: osdmap e45: 7 total, 6 up, 7 in 2024-04-15T21:33:12.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:11 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-15T21:33:12.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:11 smithi083 ceph-mon[24790]: from='osd.6 [v2:172.21.15.139:6816/121728892,v1:172.21.15.139:6817/121728892]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2024-04-15T21:33:12.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:11 smithi083 ceph-mon[24790]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2024-04-15T21:33:12.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:12 smithi139 ceph-mon[26919]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2024-04-15T21:33:12.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:12 smithi139 ceph-mon[26919]: osdmap e46: 7 total, 6 up, 7 in 2024-04-15T21:33:12.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:12 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-15T21:33:12.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:12 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-15T21:33:12.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:12 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:12.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:12 smithi139 ceph-mon[26919]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:33:12.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:12 smithi139 ceph-mon[26919]: osd.6 [v2:172.21.15.139:6816/121728892,v1:172.21.15.139:6817/121728892] boot 2024-04-15T21:33:12.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:12 smithi139 ceph-mon[26919]: osdmap e47: 7 total, 7 up, 7 in 2024-04-15T21:33:12.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:12 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-15T21:33:13.065 INFO:teuthology.orchestra.run.smithi139.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-15T21:33:13.066 INFO:teuthology.orchestra.run.smithi139.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-15T21:33:13.066 INFO:teuthology.orchestra.run.smithi139.stderr: stderr: 10+0 records in 2024-04-15T21:33:13.066 INFO:teuthology.orchestra.run.smithi139.stderr:10+0 records out 2024-04-15T21:33:13.066 INFO:teuthology.orchestra.run.smithi139.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0135021 s, 777 MB/s 2024-04-15T21:33:13.066 INFO:teuthology.orchestra.run.smithi139.stderr: stderr: 2024-04-15T21:33:13.066 INFO:teuthology.orchestra.run.smithi139.stderr:--> Zapping successful for: 2024-04-15T21:33:13.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:12 smithi083 ceph-mon[24790]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2024-04-15T21:33:13.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:12 smithi083 ceph-mon[24790]: osdmap e46: 7 total, 6 up, 7 in 2024-04-15T21:33:13.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:12 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-15T21:33:13.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:12 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-15T21:33:13.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:12 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:13.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:12 smithi083 ceph-mon[24790]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-15T21:33:13.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:12 smithi083 ceph-mon[24790]: osd.6 [v2:172.21.15.139:6816/121728892,v1:172.21.15.139:6817/121728892] boot 2024-04-15T21:33:13.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:12 smithi083 ceph-mon[24790]: osdmap e47: 7 total, 7 up, 7 in 2024-04-15T21:33:13.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:12 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-15T21:33:14.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:13 smithi083 ceph-mon[24790]: purged_snaps scrub starts 2024-04-15T21:33:14.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:13 smithi083 ceph-mon[24790]: purged_snaps scrub ok 2024-04-15T21:33:14.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:13 smithi083 ceph-mon[24790]: osdmap e48: 7 total, 7 up, 7 in 2024-04-15T21:33:14.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:13 smithi139 ceph-mon[26919]: purged_snaps scrub starts 2024-04-15T21:33:14.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:13 smithi139 ceph-mon[26919]: purged_snaps scrub ok 2024-04-15T21:33:14.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:13 smithi139 ceph-mon[26919]: osdmap e48: 7 total, 7 up, 7 in 2024-04-15T21:33:14.616 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph orch daemon add osd smithi139:vg_nvme/lv_1 2024-04-15T21:33:15.806 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:15 smithi139 ceph-mon[26919]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-15T21:33:15.806 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:15 smithi139 ceph-mon[26919]: osdmap e49: 7 total, 7 up, 7 in 2024-04-15T21:33:15.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:15 smithi083 ceph-mon[24790]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-15T21:33:15.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:15 smithi083 ceph-mon[24790]: osdmap e49: 7 total, 7 up, 7 in 2024-04-15T21:33:17.045 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:16 smithi139 ceph-mon[26919]: osdmap e50: 7 total, 7 up, 7 in 2024-04-15T21:33:17.045 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:16 smithi139 ceph-mon[26919]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:33:17.046 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-15T21:33:17.046 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-15T21:33:17.046 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:33:17.046 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:16 smithi139 ceph-mon[26919]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-15T21:33:17.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:16 smithi083 ceph-mon[24790]: osdmap e50: 7 total, 7 up, 7 in 2024-04-15T21:33:17.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:16 smithi083 ceph-mon[24790]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:33:17.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-15T21:33:17.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-15T21:33:17.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:33:17.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:16 smithi083 ceph-mon[24790]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-15T21:33:18.018 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:17 smithi139 ceph-mon[26919]: osdmap e51: 7 total, 7 up, 7 in 2024-04-15T21:33:18.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:17 smithi083 ceph-mon[24790]: osdmap e51: 7 total, 7 up, 7 in 2024-04-15T21:33:19.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:19 smithi083 ceph-mon[24790]: Detected new or changed devices on smithi139 2024-04-15T21:33:19.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:19.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:33:19.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:33:19.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:33:19.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:19 smithi083 ceph-mon[24790]: Adjusting osd_memory_target on smithi139 to 5312M 2024-04-15T21:33:19.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:19.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:19.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:33:19.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:33:19.352 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:33:19.352 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:19 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:19.352 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:19 smithi083 ceph-mon[24790]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-04-15T21:33:19.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:19 smithi139 ceph-mon[26919]: Detected new or changed devices on smithi139 2024-04-15T21:33:19.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:19.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:33:19.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:33:19.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:33:19.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:19 smithi139 ceph-mon[26919]: Adjusting osd_memory_target on smithi139 to 5312M 2024-04-15T21:33:19.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:19.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:19.364 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:33:19.364 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:33:19.364 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:33:19.364 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:19 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:19.364 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:19 smithi139 ceph-mon[26919]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-04-15T21:33:20.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:20 smithi083 ceph-mon[24790]: from='client.? 172.21.15.139:0/3158960720' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "279cc9ec-767d-4c59-9263-26db2e0e21eb"}]: dispatch 2024-04-15T21:33:20.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:20 smithi083 ceph-mon[24790]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "279cc9ec-767d-4c59-9263-26db2e0e21eb"}]: dispatch 2024-04-15T21:33:20.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:20 smithi083 ceph-mon[24790]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "279cc9ec-767d-4c59-9263-26db2e0e21eb"}]': finished 2024-04-15T21:33:20.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:20 smithi083 ceph-mon[24790]: osdmap e52: 8 total, 7 up, 8 in 2024-04-15T21:33:20.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:20 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-15T21:33:20.351 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:20 smithi083 ceph-mon[24790]: from='client.? 172.21.15.139:0/4245906597' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-15T21:33:20.362 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:20 smithi139 ceph-mon[26919]: from='client.? 172.21.15.139:0/3158960720' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "279cc9ec-767d-4c59-9263-26db2e0e21eb"}]: dispatch 2024-04-15T21:33:20.362 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:20 smithi139 ceph-mon[26919]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "279cc9ec-767d-4c59-9263-26db2e0e21eb"}]: dispatch 2024-04-15T21:33:20.362 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:20 smithi139 ceph-mon[26919]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "279cc9ec-767d-4c59-9263-26db2e0e21eb"}]': finished 2024-04-15T21:33:20.362 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:20 smithi139 ceph-mon[26919]: osdmap e52: 8 total, 7 up, 8 in 2024-04-15T21:33:20.362 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:20 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-15T21:33:20.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:20 smithi139 ceph-mon[26919]: from='client.? 172.21.15.139:0/4245906597' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-15T21:33:21.350 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:21 smithi083 ceph-mon[24790]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-15T21:33:21.362 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:21 smithi139 ceph-mon[26919]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-15T21:33:23.698 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:23 smithi139 ceph-mon[26919]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-04-15T21:33:23.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:23 smithi083 ceph-mon[24790]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-04-15T21:33:25.843 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:25 smithi139 ceph-mon[26919]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-15T21:33:25.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:25 smithi083 ceph-mon[24790]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-15T21:33:26.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:26 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-15T21:33:26.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:26 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:33:26.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:26 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-15T21:33:26.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:26 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:33:27.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:27 smithi083 ceph-mon[24790]: Deploying daemon osd.7 on smithi139 2024-04-15T21:33:27.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:27 smithi083 ceph-mon[24790]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-04-15T21:33:27.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:27 smithi139 ceph-mon[26919]: Deploying daemon osd.7 on smithi139 2024-04-15T21:33:27.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:27 smithi139 ceph-mon[26919]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-04-15T21:33:28.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:33:28.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:33:28.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:33:28.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:33:29.846 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:29 smithi139 ceph-mon[26919]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-15T21:33:29.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:29 smithi083 ceph-mon[24790]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-15T21:33:31.685 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:31 smithi139 ceph-mon[26919]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-15T21:33:31.685 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:31 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:31.685 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:31 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:33:31.685 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:31 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:33:31.685 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:31 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:33:31.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:31 smithi083 ceph-mon[24790]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-15T21:33:31.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:31 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:31.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:31 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:33:31.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:31 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:33:31.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:31 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:33:33.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:33 smithi083 ceph-mon[24790]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-15T21:33:33.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:33 smithi139 ceph-mon[26919]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-15T21:33:34.373 INFO:teuthology.orchestra.run.smithi139.stdout:Created osd(s) 7 on host 'smithi139' 2024-04-15T21:33:34.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:34 smithi083 ceph-mon[24790]: from='osd.7 [v2:172.21.15.139:6824/407497195,v1:172.21.15.139:6825/407497195]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-15T21:33:34.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:34 smithi083 ceph-mon[24790]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-15T21:33:34.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:34 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:34.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:34 smithi139 ceph-mon[26919]: from='osd.7 [v2:172.21.15.139:6824/407497195,v1:172.21.15.139:6825/407497195]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-15T21:33:34.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:34 smithi139 ceph-mon[26919]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-15T21:33:34.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:34 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:34.944 DEBUG:teuthology.orchestra.run.smithi139:osd.7> sudo journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.7.service 2024-04-15T21:33:34.946 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-15T21:33:34.947 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph osd stat -f json 2024-04-15T21:33:35.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:35 smithi083 ceph-mon[24790]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-15T21:33:35.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:35 smithi083 ceph-mon[24790]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-15T21:33:35.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:35 smithi083 ceph-mon[24790]: osdmap e53: 8 total, 7 up, 8 in 2024-04-15T21:33:35.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:35 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-15T21:33:35.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:35 smithi083 ceph-mon[24790]: from='osd.7 [v2:172.21.15.139:6824/407497195,v1:172.21.15.139:6825/407497195]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2024-04-15T21:33:35.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:35 smithi083 ceph-mon[24790]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2024-04-15T21:33:35.865 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:35 smithi139 ceph-mon[26919]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-15T21:33:35.865 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:35 smithi139 ceph-mon[26919]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-15T21:33:35.865 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:35 smithi139 ceph-mon[26919]: osdmap e53: 8 total, 7 up, 8 in 2024-04-15T21:33:35.865 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:35 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-15T21:33:35.866 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:35 smithi139 ceph-mon[26919]: from='osd.7 [v2:172.21.15.139:6824/407497195,v1:172.21.15.139:6825/407497195]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2024-04-15T21:33:35.866 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:35 smithi139 ceph-mon[26919]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2024-04-15T21:33:35.866 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:33:35 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:33:35.607+0000 7f64e4a08700 -1 osd.7 0 waiting for initial osdmap 2024-04-15T21:33:35.866 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:33:35 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:33:35.615+0000 7f64e0ba2700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-15T21:33:36.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:36 smithi083 ceph-mon[24790]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2024-04-15T21:33:36.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:36 smithi083 ceph-mon[24790]: osdmap e54: 8 total, 7 up, 8 in 2024-04-15T21:33:36.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:36 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-15T21:33:36.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:36 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-15T21:33:36.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:36 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:36.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:36 smithi139 ceph-mon[26919]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2024-04-15T21:33:36.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:36 smithi139 ceph-mon[26919]: osdmap e54: 8 total, 7 up, 8 in 2024-04-15T21:33:36.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:36 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-15T21:33:36.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:36 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-15T21:33:36.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:36 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:37.132 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:33:37.718 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:37 smithi083 ceph-mon[24790]: purged_snaps scrub starts 2024-04-15T21:33:37.718 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:37 smithi083 ceph-mon[24790]: purged_snaps scrub ok 2024-04-15T21:33:37.718 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:37 smithi083 ceph-mon[24790]: pgmap v166: 1 pgs: 1 peering; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-15T21:33:37.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:37 smithi083 ceph-mon[24790]: osd.7 [v2:172.21.15.139:6824/407497195,v1:172.21.15.139:6825/407497195] boot 2024-04-15T21:33:37.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:37 smithi083 ceph-mon[24790]: osdmap e55: 8 total, 8 up, 8 in 2024-04-15T21:33:37.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:37 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-15T21:33:37.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:37 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1086923003' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-15T21:33:37.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:37 smithi083 ceph-mon[24790]: osdmap e56: 8 total, 8 up, 8 in 2024-04-15T21:33:37.743 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1713216816,"num_in_osds":8,"osd_in_since":1713216799,"num_remapped_pgs":1} 2024-04-15T21:33:37.743 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph osd dump --format=json 2024-04-15T21:33:37.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:37 smithi139 ceph-mon[26919]: purged_snaps scrub starts 2024-04-15T21:33:37.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:37 smithi139 ceph-mon[26919]: purged_snaps scrub ok 2024-04-15T21:33:37.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:37 smithi139 ceph-mon[26919]: pgmap v166: 1 pgs: 1 peering; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-15T21:33:37.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:37 smithi139 ceph-mon[26919]: osd.7 [v2:172.21.15.139:6824/407497195,v1:172.21.15.139:6825/407497195] boot 2024-04-15T21:33:37.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:37 smithi139 ceph-mon[26919]: osdmap e55: 8 total, 8 up, 8 in 2024-04-15T21:33:37.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:37 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-15T21:33:37.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:37 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/1086923003' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-15T21:33:37.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:37 smithi139 ceph-mon[26919]: osdmap e56: 8 total, 8 up, 8 in 2024-04-15T21:33:37.970 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:33:39.626 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:33:39.626 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":58,"fsid":"e4a42038-fb6e-11ee-bc8f-c7b262605968","created":"2024-04-15T21:27:37.736474+0000","modified":"2024-04-15T21:33:39.530383+0000","last_up_change":"2024-04-15T21:33:36.602112+0000","last_in_change":"2024-04-15T21:33:19.475311+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-15T21:31:34.794440+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"4384484b-f651-428f-a059-88f15e28d2d4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":1609792529},{"type":"v1","addr":"172.21.15.83:6803","nonce":1609792529}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":1609792529},{"type":"v1","addr":"172.21.15.83:6805","nonce":1609792529}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":1609792529},{"type":"v1","addr":"172.21.15.83:6809","nonce":1609792529}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":1609792529},{"type":"v1","addr":"172.21.15.83:6807","nonce":1609792529}]},"public_addr":"172.21.15.83:6803/1609792529","cluster_addr":"172.21.15.83:6805/1609792529","heartbeat_back_addr":"172.21.15.83:6809/1609792529","heartbeat_front_addr":"172.21.15.83:6807/1609792529","state":["exists","up"]},{"osd":1,"uuid":"886cd293-b724-4daa-81c2-aa6d6dc9821a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":3642014361},{"type":"v1","addr":"172.21.15.83:6811","nonce":3642014361}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":3642014361},{"type":"v1","addr":"172.21.15.83:6813","nonce":3642014361}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":3642014361},{"type":"v1","addr":"172.21.15.83:6817","nonce":3642014361}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":3642014361},{"type":"v1","addr":"172.21.15.83:6815","nonce":3642014361}]},"public_addr":"172.21.15.83:6811/3642014361","cluster_addr":"172.21.15.83:6813/3642014361","heartbeat_back_addr":"172.21.15.83:6817/3642014361","heartbeat_front_addr":"172.21.15.83:6815/3642014361","state":["exists","up"]},{"osd":2,"uuid":"fddc5d10-da39-4a82-87cc-36e35a7e5601","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":2309901275},{"type":"v1","addr":"172.21.15.83:6819","nonce":2309901275}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":2309901275},{"type":"v1","addr":"172.21.15.83:6821","nonce":2309901275}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":2309901275},{"type":"v1","addr":"172.21.15.83:6825","nonce":2309901275}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":2309901275},{"type":"v1","addr":"172.21.15.83:6823","nonce":2309901275}]},"public_addr":"172.21.15.83:6819/2309901275","cluster_addr":"172.21.15.83:6821/2309901275","heartbeat_back_addr":"172.21.15.83:6825/2309901275","heartbeat_front_addr":"172.21.15.83:6823/2309901275","state":["exists","up"]},{"osd":3,"uuid":"8ea0f736-4b9b-4015-a82b-dcf9278d51b4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":2705135586},{"type":"v1","addr":"172.21.15.83:6827","nonce":2705135586}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":2705135586},{"type":"v1","addr":"172.21.15.83:6829","nonce":2705135586}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":2705135586},{"type":"v1","addr":"172.21.15.83:6833","nonce":2705135586}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":2705135586},{"type":"v1","addr":"172.21.15.83:6831","nonce":2705135586}]},"public_addr":"172.21.15.83:6827/2705135586","cluster_addr":"172.21.15.83:6829/2705135586","heartbeat_back_addr":"172.21.15.83:6833/2705135586","heartbeat_front_addr":"172.21.15.83:6831/2705135586","state":["exists","up"]},{"osd":4,"uuid":"9a9d5120-0512-44aa-b6f4-8e70170e861f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6800","nonce":962351193},{"type":"v1","addr":"172.21.15.139:6801","nonce":962351193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6802","nonce":962351193},{"type":"v1","addr":"172.21.15.139:6803","nonce":962351193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6806","nonce":962351193},{"type":"v1","addr":"172.21.15.139:6807","nonce":962351193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6804","nonce":962351193},{"type":"v1","addr":"172.21.15.139:6805","nonce":962351193}]},"public_addr":"172.21.15.139:6801/962351193","cluster_addr":"172.21.15.139:6803/962351193","heartbeat_back_addr":"172.21.15.139:6807/962351193","heartbeat_front_addr":"172.21.15.139:6805/962351193","state":["exists","up"]},{"osd":5,"uuid":"fd246cf7-ef0b-4cca-85ce-3dfbc73d75a2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6808","nonce":2653213923},{"type":"v1","addr":"172.21.15.139:6809","nonce":2653213923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6810","nonce":2653213923},{"type":"v1","addr":"172.21.15.139:6811","nonce":2653213923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6814","nonce":2653213923},{"type":"v1","addr":"172.21.15.139:6815","nonce":2653213923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6812","nonce":2653213923},{"type":"v1","addr":"172.21.15.139:6813","nonce":2653213923}]},"public_addr":"172.21.15.139:6809/2653213923","cluster_addr":"172.21.15.139:6811/2653213923","heartbeat_back_addr":"172.21.15.139:6815/2653213923","heartbeat_front_addr":"172.21.15.139:6813/2653213923","state":["exists","up"]},{"osd":6,"uuid":"fed409f9-7882-4a83-8667-79afe110ad61","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6816","nonce":121728892},{"type":"v1","addr":"172.21.15.139:6817","nonce":121728892}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6818","nonce":121728892},{"type":"v1","addr":"172.21.15.139:6819","nonce":121728892}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6822","nonce":121728892},{"type":"v1","addr":"172.21.15.139:6823","nonce":121728892}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6820","nonce":121728892},{"type":"v1","addr":"172.21.15.139:6821","nonce":121728892}]},"public_addr":"172.21.15.139:6817/121728892","cluster_addr":"172.21.15.139:6819/121728892","heartbeat_back_addr":"172.21.15.139:6823/121728892","heartbeat_front_addr":"172.21.15.139:6821/121728892","state":["exists","up"]},{"osd":7,"uuid":"279cc9ec-767d-4c59-9263-26db2e0e21eb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6824","nonce":407497195},{"type":"v1","addr":"172.21.15.139:6825","nonce":407497195}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6826","nonce":407497195},{"type":"v1","addr":"172.21.15.139:6827","nonce":407497195}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6830","nonce":407497195},{"type":"v1","addr":"172.21.15.139:6831","nonce":407497195}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6828","nonce":407497195},{"type":"v1","addr":"172.21.15.139:6829","nonce":407497195}]},"public_addr":"172.21.15.139:6825/407497195","cluster_addr":"172.21.15.139:6827/407497195","heartbeat_back_addr":"172.21.15.139:6831/407497195","heartbeat_front_addr":"172.21.15.139:6829/407497195","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:30:34.198700+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:31:03.486361+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:31:32.123617+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:31:59.681126+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:32:22.489991+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:32:46.406131+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:33:10.823461+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:33:35.200554+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:0/3779329394":"2024-04-16T21:29:27.507590+0000","172.21.15.83:0/536021479":"2024-04-16T21:28:37.569239+0000","172.21.15.83:0/962641305":"2024-04-16T21:29:27.507590+0000","172.21.15.83:0/293924267":"2024-04-16T21:28:37.569239+0000","172.21.15.83:0/1471685484":"2024-04-16T21:28:37.569239+0000","172.21.15.83:6801/1859356746":"2024-04-16T21:29:27.507590+0000","172.21.15.83:6800/1859356746":"2024-04-16T21:29:27.507590+0000","172.21.15.83:6801/187728414":"2024-04-16T21:28:37.569239+0000","172.21.15.83:0/677357066":"2024-04-16T21:28:08.166307+0000","172.21.15.83:0/3350288289":"2024-04-16T21:28:08.166307+0000","172.21.15.83:6800/531778747":"2024-04-16T21:28:08.166307+0000","172.21.15.83:0/748536671":"2024-04-16T21:29:27.507590+0000","172.21.15.83:6800/187728414":"2024-04-16T21:28:37.569239+0000","172.21.15.83:6801/531778747":"2024-04-16T21:28:08.166307+0000","172.21.15.83:0/3534488545":"2024-04-16T21:28:08.166307+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-15T21:33:39.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:39 smithi083 ceph-mon[24790]: pgmap v169: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:33:39.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:39 smithi083 ceph-mon[24790]: osdmap e57: 8 total, 8 up, 8 in 2024-04-15T21:33:39.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:39.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:33:39.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:33:39.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:33:39.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:33:39.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:39.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:39.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:33:39.852 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:33:39.852 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:33:39.852 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:39.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:39 smithi139 ceph-mon[26919]: pgmap v169: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:33:39.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:39 smithi139 ceph-mon[26919]: osdmap e57: 8 total, 8 up, 8 in 2024-04-15T21:33:39.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:39.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:33:39.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:33:39.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:33:39.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-15T21:33:39.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:39.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:39.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:33:39.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:33:39.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:33:39.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:33:40.413 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-15T21:31:34.794440+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-15T21:33:40.413 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-15T21:33:40.634 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:33:40.756 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:40 smithi083 ceph-mon[24790]: Detected new or changed devices on smithi139 2024-04-15T21:33:40.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:40 smithi083 ceph-mon[24790]: Adjusting osd_memory_target on smithi139 to 3984M 2024-04-15T21:33:40.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:40 smithi083 ceph-mon[24790]: osdmap e58: 8 total, 8 up, 8 in 2024-04-15T21:33:40.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:40 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/3269626374' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-15T21:33:40.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:40 smithi139 ceph-mon[26919]: Detected new or changed devices on smithi139 2024-04-15T21:33:40.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:40 smithi139 ceph-mon[26919]: Adjusting osd_memory_target on smithi139 to 3984M 2024-04-15T21:33:40.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:40 smithi139 ceph-mon[26919]: osdmap e58: 8 total, 8 up, 8 in 2024-04-15T21:33:40.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:40 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/3269626374' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-15T21:33:41.822 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:41 smithi083 ceph-mon[24790]: pgmap v172: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:33:41.822 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:41 smithi083 ceph-mon[24790]: osdmap e59: 8 total, 8 up, 8 in 2024-04-15T21:33:41.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:41 smithi139 ceph-mon[26919]: pgmap v172: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:33:41.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:41 smithi139 ceph-mon[26919]: osdmap e59: 8 total, 8 up, 8 in 2024-04-15T21:33:42.460 INFO:teuthology.orchestra.run.smithi083.stdout:pg_num: 1 2024-04-15T21:33:42.789 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:42 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/3409225112' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-15T21:33:42.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:42 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/3409225112' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-15T21:33:43.153 INFO:tasks.cephadm:Setting up client nodes... 2024-04-15T21:33:43.153 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-15T21:33:43.798 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:43 smithi083 ceph-mon[24790]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-04-15T21:33:43.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:43 smithi139 ceph-mon[26919]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-04-15T21:33:45.168 INFO:teuthology.orchestra.run.smithi083.stdout:[client.0] 2024-04-15T21:33:45.168 INFO:teuthology.orchestra.run.smithi083.stdout: key = AQA5nR1mCOK8CRAADbGX2wjoI+vRkrucDxt4vQ== 2024-04-15T21:33:45.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:45 smithi083 ceph-mon[24790]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-15T21:33:45.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:45 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/2681247229' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-15T21:33:45.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:45 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/2681247229' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-15T21:33:45.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:45 smithi139 ceph-mon[26919]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-15T21:33:45.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:45 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/2681247229' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-15T21:33:45.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:45 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/2681247229' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-15T21:33:45.984 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-15T21:33:45.984 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-15T21:33:45.984 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-15T21:33:46.031 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-15T21:33:46.031 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-15T21:33:46.031 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph mgr dump --format=json 2024-04-15T21:33:46.283 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:33:47.781 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:47 smithi083 ceph-mon[24790]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-15T21:33:47.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:47 smithi139 ceph-mon[26919]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-15T21:33:48.207 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:33:48.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:48 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1468153991' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-15T21:33:48.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:48 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/1468153991' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-15T21:33:48.887 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":19,"active_gid":14200,"active_name":"smithi083.kxqarn","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6800","nonce":2448111662},{"type":"v1","addr":"172.21.15.83:6801","nonce":2448111662}]},"active_addr":"172.21.15.83:6801/2448111662","active_change":"2024-04-15T21:29:27.507906+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14220,"name":"smithi139.bqqykn","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.83:8443/","prometheus":"http://172.21.15.83:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":2832366938}]},{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":41048382}]},{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":4208133363}]},{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":3513750778}]}]}} 2024-04-15T21:33:48.893 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-15T21:33:48.894 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-15T21:33:48.894 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph osd dump --format=json 2024-04-15T21:33:49.110 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:33:49.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:49 smithi083 ceph-mon[24790]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-04-15T21:33:49.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:49 smithi139 ceph-mon[26919]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-04-15T21:33:51.112 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:33:51.112 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":59,"fsid":"e4a42038-fb6e-11ee-bc8f-c7b262605968","created":"2024-04-15T21:27:37.736474+0000","modified":"2024-04-15T21:33:40.540127+0000","last_up_change":"2024-04-15T21:33:36.602112+0000","last_in_change":"2024-04-15T21:33:19.475311+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-15T21:31:34.794440+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"4384484b-f651-428f-a059-88f15e28d2d4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":1609792529},{"type":"v1","addr":"172.21.15.83:6803","nonce":1609792529}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":1609792529},{"type":"v1","addr":"172.21.15.83:6805","nonce":1609792529}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":1609792529},{"type":"v1","addr":"172.21.15.83:6809","nonce":1609792529}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":1609792529},{"type":"v1","addr":"172.21.15.83:6807","nonce":1609792529}]},"public_addr":"172.21.15.83:6803/1609792529","cluster_addr":"172.21.15.83:6805/1609792529","heartbeat_back_addr":"172.21.15.83:6809/1609792529","heartbeat_front_addr":"172.21.15.83:6807/1609792529","state":["exists","up"]},{"osd":1,"uuid":"886cd293-b724-4daa-81c2-aa6d6dc9821a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":3642014361},{"type":"v1","addr":"172.21.15.83:6811","nonce":3642014361}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":3642014361},{"type":"v1","addr":"172.21.15.83:6813","nonce":3642014361}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":3642014361},{"type":"v1","addr":"172.21.15.83:6817","nonce":3642014361}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":3642014361},{"type":"v1","addr":"172.21.15.83:6815","nonce":3642014361}]},"public_addr":"172.21.15.83:6811/3642014361","cluster_addr":"172.21.15.83:6813/3642014361","heartbeat_back_addr":"172.21.15.83:6817/3642014361","heartbeat_front_addr":"172.21.15.83:6815/3642014361","state":["exists","up"]},{"osd":2,"uuid":"fddc5d10-da39-4a82-87cc-36e35a7e5601","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":2309901275},{"type":"v1","addr":"172.21.15.83:6819","nonce":2309901275}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":2309901275},{"type":"v1","addr":"172.21.15.83:6821","nonce":2309901275}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":2309901275},{"type":"v1","addr":"172.21.15.83:6825","nonce":2309901275}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":2309901275},{"type":"v1","addr":"172.21.15.83:6823","nonce":2309901275}]},"public_addr":"172.21.15.83:6819/2309901275","cluster_addr":"172.21.15.83:6821/2309901275","heartbeat_back_addr":"172.21.15.83:6825/2309901275","heartbeat_front_addr":"172.21.15.83:6823/2309901275","state":["exists","up"]},{"osd":3,"uuid":"8ea0f736-4b9b-4015-a82b-dcf9278d51b4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":2705135586},{"type":"v1","addr":"172.21.15.83:6827","nonce":2705135586}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":2705135586},{"type":"v1","addr":"172.21.15.83:6829","nonce":2705135586}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":2705135586},{"type":"v1","addr":"172.21.15.83:6833","nonce":2705135586}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":2705135586},{"type":"v1","addr":"172.21.15.83:6831","nonce":2705135586}]},"public_addr":"172.21.15.83:6827/2705135586","cluster_addr":"172.21.15.83:6829/2705135586","heartbeat_back_addr":"172.21.15.83:6833/2705135586","heartbeat_front_addr":"172.21.15.83:6831/2705135586","state":["exists","up"]},{"osd":4,"uuid":"9a9d5120-0512-44aa-b6f4-8e70170e861f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6800","nonce":962351193},{"type":"v1","addr":"172.21.15.139:6801","nonce":962351193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6802","nonce":962351193},{"type":"v1","addr":"172.21.15.139:6803","nonce":962351193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6806","nonce":962351193},{"type":"v1","addr":"172.21.15.139:6807","nonce":962351193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6804","nonce":962351193},{"type":"v1","addr":"172.21.15.139:6805","nonce":962351193}]},"public_addr":"172.21.15.139:6801/962351193","cluster_addr":"172.21.15.139:6803/962351193","heartbeat_back_addr":"172.21.15.139:6807/962351193","heartbeat_front_addr":"172.21.15.139:6805/962351193","state":["exists","up"]},{"osd":5,"uuid":"fd246cf7-ef0b-4cca-85ce-3dfbc73d75a2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6808","nonce":2653213923},{"type":"v1","addr":"172.21.15.139:6809","nonce":2653213923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6810","nonce":2653213923},{"type":"v1","addr":"172.21.15.139:6811","nonce":2653213923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6814","nonce":2653213923},{"type":"v1","addr":"172.21.15.139:6815","nonce":2653213923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6812","nonce":2653213923},{"type":"v1","addr":"172.21.15.139:6813","nonce":2653213923}]},"public_addr":"172.21.15.139:6809/2653213923","cluster_addr":"172.21.15.139:6811/2653213923","heartbeat_back_addr":"172.21.15.139:6815/2653213923","heartbeat_front_addr":"172.21.15.139:6813/2653213923","state":["exists","up"]},{"osd":6,"uuid":"fed409f9-7882-4a83-8667-79afe110ad61","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6816","nonce":121728892},{"type":"v1","addr":"172.21.15.139:6817","nonce":121728892}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6818","nonce":121728892},{"type":"v1","addr":"172.21.15.139:6819","nonce":121728892}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6822","nonce":121728892},{"type":"v1","addr":"172.21.15.139:6823","nonce":121728892}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6820","nonce":121728892},{"type":"v1","addr":"172.21.15.139:6821","nonce":121728892}]},"public_addr":"172.21.15.139:6817/121728892","cluster_addr":"172.21.15.139:6819/121728892","heartbeat_back_addr":"172.21.15.139:6823/121728892","heartbeat_front_addr":"172.21.15.139:6821/121728892","state":["exists","up"]},{"osd":7,"uuid":"279cc9ec-767d-4c59-9263-26db2e0e21eb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6824","nonce":407497195},{"type":"v1","addr":"172.21.15.139:6825","nonce":407497195}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6826","nonce":407497195},{"type":"v1","addr":"172.21.15.139:6827","nonce":407497195}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6830","nonce":407497195},{"type":"v1","addr":"172.21.15.139:6831","nonce":407497195}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6828","nonce":407497195},{"type":"v1","addr":"172.21.15.139:6829","nonce":407497195}]},"public_addr":"172.21.15.139:6825/407497195","cluster_addr":"172.21.15.139:6827/407497195","heartbeat_back_addr":"172.21.15.139:6831/407497195","heartbeat_front_addr":"172.21.15.139:6829/407497195","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:30:34.198700+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:31:03.486361+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:31:32.123617+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:31:59.681126+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:32:22.489991+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:32:46.406131+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:33:10.823461+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:33:35.200554+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:0/3779329394":"2024-04-16T21:29:27.507590+0000","172.21.15.83:0/536021479":"2024-04-16T21:28:37.569239+0000","172.21.15.83:0/962641305":"2024-04-16T21:29:27.507590+0000","172.21.15.83:0/293924267":"2024-04-16T21:28:37.569239+0000","172.21.15.83:0/1471685484":"2024-04-16T21:28:37.569239+0000","172.21.15.83:6801/1859356746":"2024-04-16T21:29:27.507590+0000","172.21.15.83:6800/1859356746":"2024-04-16T21:29:27.507590+0000","172.21.15.83:6801/187728414":"2024-04-16T21:28:37.569239+0000","172.21.15.83:0/677357066":"2024-04-16T21:28:08.166307+0000","172.21.15.83:0/3350288289":"2024-04-16T21:28:08.166307+0000","172.21.15.83:6800/531778747":"2024-04-16T21:28:08.166307+0000","172.21.15.83:0/748536671":"2024-04-16T21:29:27.507590+0000","172.21.15.83:6800/187728414":"2024-04-16T21:28:37.569239+0000","172.21.15.83:6801/531778747":"2024-04-16T21:28:08.166307+0000","172.21.15.83:0/3534488545":"2024-04-16T21:28:08.166307+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-15T21:33:51.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:51 smithi083 ceph-mon[24790]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-15T21:33:51.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:51 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/3948164548' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-15T21:33:51.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:51 smithi139 ceph-mon[26919]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-15T21:33:51.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:51 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/3948164548' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-15T21:33:51.897 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-15T21:33:51.897 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph osd dump --format=json 2024-04-15T21:33:52.115 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:33:53.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:53 smithi083 ceph-mon[24790]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-15T21:33:53.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:53 smithi139 ceph-mon[26919]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-15T21:33:53.882 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:33:53.882 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":59,"fsid":"e4a42038-fb6e-11ee-bc8f-c7b262605968","created":"2024-04-15T21:27:37.736474+0000","modified":"2024-04-15T21:33:40.540127+0000","last_up_change":"2024-04-15T21:33:36.602112+0000","last_in_change":"2024-04-15T21:33:19.475311+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-15T21:31:34.794440+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"4384484b-f651-428f-a059-88f15e28d2d4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":1609792529},{"type":"v1","addr":"172.21.15.83:6803","nonce":1609792529}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":1609792529},{"type":"v1","addr":"172.21.15.83:6805","nonce":1609792529}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":1609792529},{"type":"v1","addr":"172.21.15.83:6809","nonce":1609792529}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":1609792529},{"type":"v1","addr":"172.21.15.83:6807","nonce":1609792529}]},"public_addr":"172.21.15.83:6803/1609792529","cluster_addr":"172.21.15.83:6805/1609792529","heartbeat_back_addr":"172.21.15.83:6809/1609792529","heartbeat_front_addr":"172.21.15.83:6807/1609792529","state":["exists","up"]},{"osd":1,"uuid":"886cd293-b724-4daa-81c2-aa6d6dc9821a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":3642014361},{"type":"v1","addr":"172.21.15.83:6811","nonce":3642014361}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":3642014361},{"type":"v1","addr":"172.21.15.83:6813","nonce":3642014361}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":3642014361},{"type":"v1","addr":"172.21.15.83:6817","nonce":3642014361}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":3642014361},{"type":"v1","addr":"172.21.15.83:6815","nonce":3642014361}]},"public_addr":"172.21.15.83:6811/3642014361","cluster_addr":"172.21.15.83:6813/3642014361","heartbeat_back_addr":"172.21.15.83:6817/3642014361","heartbeat_front_addr":"172.21.15.83:6815/3642014361","state":["exists","up"]},{"osd":2,"uuid":"fddc5d10-da39-4a82-87cc-36e35a7e5601","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":2309901275},{"type":"v1","addr":"172.21.15.83:6819","nonce":2309901275}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":2309901275},{"type":"v1","addr":"172.21.15.83:6821","nonce":2309901275}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":2309901275},{"type":"v1","addr":"172.21.15.83:6825","nonce":2309901275}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":2309901275},{"type":"v1","addr":"172.21.15.83:6823","nonce":2309901275}]},"public_addr":"172.21.15.83:6819/2309901275","cluster_addr":"172.21.15.83:6821/2309901275","heartbeat_back_addr":"172.21.15.83:6825/2309901275","heartbeat_front_addr":"172.21.15.83:6823/2309901275","state":["exists","up"]},{"osd":3,"uuid":"8ea0f736-4b9b-4015-a82b-dcf9278d51b4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":2705135586},{"type":"v1","addr":"172.21.15.83:6827","nonce":2705135586}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":2705135586},{"type":"v1","addr":"172.21.15.83:6829","nonce":2705135586}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":2705135586},{"type":"v1","addr":"172.21.15.83:6833","nonce":2705135586}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":2705135586},{"type":"v1","addr":"172.21.15.83:6831","nonce":2705135586}]},"public_addr":"172.21.15.83:6827/2705135586","cluster_addr":"172.21.15.83:6829/2705135586","heartbeat_back_addr":"172.21.15.83:6833/2705135586","heartbeat_front_addr":"172.21.15.83:6831/2705135586","state":["exists","up"]},{"osd":4,"uuid":"9a9d5120-0512-44aa-b6f4-8e70170e861f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6800","nonce":962351193},{"type":"v1","addr":"172.21.15.139:6801","nonce":962351193}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6802","nonce":962351193},{"type":"v1","addr":"172.21.15.139:6803","nonce":962351193}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6806","nonce":962351193},{"type":"v1","addr":"172.21.15.139:6807","nonce":962351193}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6804","nonce":962351193},{"type":"v1","addr":"172.21.15.139:6805","nonce":962351193}]},"public_addr":"172.21.15.139:6801/962351193","cluster_addr":"172.21.15.139:6803/962351193","heartbeat_back_addr":"172.21.15.139:6807/962351193","heartbeat_front_addr":"172.21.15.139:6805/962351193","state":["exists","up"]},{"osd":5,"uuid":"fd246cf7-ef0b-4cca-85ce-3dfbc73d75a2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6808","nonce":2653213923},{"type":"v1","addr":"172.21.15.139:6809","nonce":2653213923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6810","nonce":2653213923},{"type":"v1","addr":"172.21.15.139:6811","nonce":2653213923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6814","nonce":2653213923},{"type":"v1","addr":"172.21.15.139:6815","nonce":2653213923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6812","nonce":2653213923},{"type":"v1","addr":"172.21.15.139:6813","nonce":2653213923}]},"public_addr":"172.21.15.139:6809/2653213923","cluster_addr":"172.21.15.139:6811/2653213923","heartbeat_back_addr":"172.21.15.139:6815/2653213923","heartbeat_front_addr":"172.21.15.139:6813/2653213923","state":["exists","up"]},{"osd":6,"uuid":"fed409f9-7882-4a83-8667-79afe110ad61","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6816","nonce":121728892},{"type":"v1","addr":"172.21.15.139:6817","nonce":121728892}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6818","nonce":121728892},{"type":"v1","addr":"172.21.15.139:6819","nonce":121728892}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6822","nonce":121728892},{"type":"v1","addr":"172.21.15.139:6823","nonce":121728892}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6820","nonce":121728892},{"type":"v1","addr":"172.21.15.139:6821","nonce":121728892}]},"public_addr":"172.21.15.139:6817/121728892","cluster_addr":"172.21.15.139:6819/121728892","heartbeat_back_addr":"172.21.15.139:6823/121728892","heartbeat_front_addr":"172.21.15.139:6821/121728892","state":["exists","up"]},{"osd":7,"uuid":"279cc9ec-767d-4c59-9263-26db2e0e21eb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6824","nonce":407497195},{"type":"v1","addr":"172.21.15.139:6825","nonce":407497195}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6826","nonce":407497195},{"type":"v1","addr":"172.21.15.139:6827","nonce":407497195}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6830","nonce":407497195},{"type":"v1","addr":"172.21.15.139:6831","nonce":407497195}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6828","nonce":407497195},{"type":"v1","addr":"172.21.15.139:6829","nonce":407497195}]},"public_addr":"172.21.15.139:6825/407497195","cluster_addr":"172.21.15.139:6827/407497195","heartbeat_back_addr":"172.21.15.139:6831/407497195","heartbeat_front_addr":"172.21.15.139:6829/407497195","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:30:34.198700+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:31:03.486361+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:31:32.123617+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:31:59.681126+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:32:22.489991+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:32:46.406131+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:33:10.823461+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-15T21:33:35.200554+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:0/3779329394":"2024-04-16T21:29:27.507590+0000","172.21.15.83:0/536021479":"2024-04-16T21:28:37.569239+0000","172.21.15.83:0/962641305":"2024-04-16T21:29:27.507590+0000","172.21.15.83:0/293924267":"2024-04-16T21:28:37.569239+0000","172.21.15.83:0/1471685484":"2024-04-16T21:28:37.569239+0000","172.21.15.83:6801/1859356746":"2024-04-16T21:29:27.507590+0000","172.21.15.83:6800/1859356746":"2024-04-16T21:29:27.507590+0000","172.21.15.83:6801/187728414":"2024-04-16T21:28:37.569239+0000","172.21.15.83:0/677357066":"2024-04-16T21:28:08.166307+0000","172.21.15.83:0/3350288289":"2024-04-16T21:28:08.166307+0000","172.21.15.83:6800/531778747":"2024-04-16T21:28:08.166307+0000","172.21.15.83:0/748536671":"2024-04-16T21:29:27.507590+0000","172.21.15.83:6800/187728414":"2024-04-16T21:28:37.569239+0000","172.21.15.83:6801/531778747":"2024-04-16T21:28:08.166307+0000","172.21.15.83:0/3534488545":"2024-04-16T21:28:08.166307+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-15T21:33:54.554 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-15T21:33:54.554 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-15T21:33:54.555 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-15T21:33:54.555 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-15T21:33:54.555 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-15T21:33:54.556 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-15T21:33:54.556 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-15T21:33:54.556 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-15T21:33:54.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:54 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/3533224278' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-15T21:33:54.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:54 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/3533224278' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-15T21:33:54.964 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:33:54.964 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:33:54.964 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:33:54.966 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:33:54.967 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:33:54.967 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:33:54.967 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:33:54.975 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:33:55.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:55 smithi083 ceph-mon[24790]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:33:55.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:55 smithi139 ceph-mon[26919]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:33:57.600 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:57 smithi083 ceph-mon[24790]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:33:57.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:57 smithi139 ceph-mon[26919]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:33:59.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:33:59 smithi083 ceph-mon[24790]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:33:59.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:33:59 smithi139 ceph-mon[26919]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:01.649 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:01 smithi083 ceph-mon[24790]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:01.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:01 smithi139 ceph-mon[26919]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:03.849 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:03 smithi083 ceph-mon[24790]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:03.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:03 smithi139 ceph-mon[26919]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:05.291 INFO:teuthology.orchestra.run.smithi083.stdout:60129542182 2024-04-15T21:34:05.292 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-15T21:34:05.822 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:05 smithi083 ceph-mon[24790]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:05.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:05 smithi139 ceph-mon[26919]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:07.798 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:07 smithi083 ceph-mon[24790]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:07.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:07 smithi139 ceph-mon[26919]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:08.850 INFO:teuthology.orchestra.run.smithi083.stdout:201863462925 2024-04-15T21:34:08.850 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-15T21:34:09.734 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:09 smithi083 ceph-mon[24790]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:09.757 INFO:teuthology.orchestra.run.smithi083.stdout:236223201288 2024-04-15T21:34:09.757 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-15T21:34:09.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:09 smithi139 ceph-mon[26919]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:10.476 INFO:teuthology.orchestra.run.smithi083.stdout:38654705710 2024-04-15T21:34:10.477 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-15T21:34:11.275 INFO:teuthology.orchestra.run.smithi083.stdout:167503724563 2024-04-15T21:34:11.276 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-15T21:34:11.325 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:34:11.751 INFO:teuthology.orchestra.run.smithi083.stdout:133143986199 2024-04-15T21:34:11.751 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-15T21:34:11.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:11 smithi139 ceph-mon[26919]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:11.906 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:11 smithi083 ceph-mon[24790]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:12.145 INFO:teuthology.orchestra.run.smithi083.stdout:111669149725 2024-04-15T21:34:12.145 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-15T21:34:12.171 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:34:12.172 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:34:12.505 INFO:teuthology.orchestra.run.smithi083.stdout:81604378659 2024-04-15T21:34:12.505 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-15T21:34:13.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:13 smithi083 ceph-mon[24790]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:13.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:13 smithi139 ceph-mon[26919]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:14.668 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:34:14.673 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:34:14.673 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:34:14.674 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:34:14.675 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:34:15.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:15 smithi083 ceph-mon[24790]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:15.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:15 smithi139 ceph-mon[26919]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:17.032 INFO:teuthology.orchestra.run.smithi083.stdout:201863462927 2024-04-15T21:34:17.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:17 smithi083 ceph-mon[24790]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:17.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:17 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/3313527424' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-15T21:34:17.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:17 smithi139 ceph-mon[26919]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:17.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:17 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/3313527424' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-15T21:34:17.933 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462925 got 201863462927 for osd.6 2024-04-15T21:34:17.933 DEBUG:teuthology.parallel:result is None 2024-04-15T21:34:19.778 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:19 smithi083 ceph-mon[24790]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:19.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:19 smithi139 ceph-mon[26919]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:20.943 INFO:teuthology.orchestra.run.smithi083.stdout:38654705712 2024-04-15T21:34:21.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:21 smithi083 ceph-mon[24790]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:21.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:21 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1156407858' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-15T21:34:22.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:21 smithi139 ceph-mon[26919]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:22.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:21 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/1156407858' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-15T21:34:22.518 INFO:teuthology.orchestra.run.smithi083.stdout:236223201291 2024-04-15T21:34:23.072 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:22 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/2044728025' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-15T21:34:23.098 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201288 got 236223201291 for osd.7 2024-04-15T21:34:23.098 DEBUG:teuthology.parallel:result is None 2024-04-15T21:34:23.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:22 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/2044728025' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-15T21:34:23.652 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:23 smithi083 ceph-mon[24790]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:24.088 INFO:teuthology.orchestra.run.smithi083.stdout:81604378661 2024-04-15T21:34:24.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:23 smithi139 ceph-mon[26919]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:24.714 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:24 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1910286354' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-15T21:34:24.844 INFO:teuthology.orchestra.run.smithi083.stdout:111669149728 2024-04-15T21:34:25.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:24 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/1910286354' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-15T21:34:25.189 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705710 got 38654705712 for osd.0 2024-04-15T21:34:25.190 DEBUG:teuthology.parallel:result is None 2024-04-15T21:34:25.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:25 smithi083 ceph-mon[24790]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:25.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:25 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/2533138695' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-15T21:34:25.978 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149725 got 111669149728 for osd.3 2024-04-15T21:34:25.978 DEBUG:teuthology.parallel:result is None 2024-04-15T21:34:26.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:25 smithi139 ceph-mon[26919]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:26.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:25 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/2533138695' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-15T21:34:26.365 INFO:teuthology.orchestra.run.smithi083.stdout:60129542186 2024-04-15T21:34:27.013 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:26 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/3603226881' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-15T21:34:27.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:26 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/3603226881' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-15T21:34:27.367 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378659 got 81604378661 for osd.2 2024-04-15T21:34:27.367 DEBUG:teuthology.parallel:result is None 2024-04-15T21:34:27.555 INFO:teuthology.orchestra.run.smithi083.stdout:167503724566 2024-04-15T21:34:27.884 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:27 smithi083 ceph-mon[24790]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:27.884 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:27 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1311628746' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-15T21:34:28.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:27 smithi139 ceph-mon[26919]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:28.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:27 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/1311628746' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-15T21:34:28.157 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724563 got 167503724566 for osd.5 2024-04-15T21:34:28.157 DEBUG:teuthology.parallel:result is None 2024-04-15T21:34:28.631 INFO:teuthology.orchestra.run.smithi083.stdout:133143986203 2024-04-15T21:34:28.736 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:34:28.736 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:34:28.736 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:28 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1809715259' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-15T21:34:28.877 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542182 got 60129542186 for osd.1 2024-04-15T21:34:28.877 DEBUG:teuthology.parallel:result is None 2024-04-15T21:34:29.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:34:29.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:34:29.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:28 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/1809715259' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-15T21:34:29.422 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986199 got 133143986203 for osd.4 2024-04-15T21:34:29.422 DEBUG:teuthology.parallel:result is None 2024-04-15T21:34:29.423 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-15T21:34:29.423 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph pg dump --format=json 2024-04-15T21:34:29.642 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:34:29.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:29 smithi083 ceph-mon[24790]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:30.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:29 smithi139 ceph-mon[26919]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:31.307 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:34:31.308 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2024-04-15T21:34:31.946 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:31 smithi083 ceph-mon[24790]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:31.971 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":198,"stamp":"2024-04-15T21:34:30.534484+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48748,"kb_used_data":6316,"kb_used_omap":0,"kb_used_meta":42368,"kb_avail":749912468,"statfs":{"total":767960285184,"available":767910367232,"internally_reserved":0,"allocated":6467584,"data_stored":3547758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43384832},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002326"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-04-15T21:33:40.554061+0000","last_change":"2024-04-15T21:33:40.554061+0000","last_active":"2024-04-15T21:33:40.554061+0000","last_peered":"2024-04-15T21:33:40.554061+0000","last_clean":"2024-04-15T21:33:40.554061+0000","last_became_active":"2024-04-15T21:33:40.553676+0000","last_became_peered":"2024-04-15T21:33:40.553676+0000","last_unstale":"2024-04-15T21:33:40.554061+0000","last_undegraded":"2024-04-15T21:33:40.554061+0000","last_fullsized":"2024-04-15T21:33:40.554061+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-15T21:31:34.794805+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-15T21:31:34.794805+0000","last_clean_scrub_stamp":"2024-04-15T21:31:34.794805+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-17T06:04:05.960190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2662400,"data_stored":2644640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201293,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.495}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61599999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.37}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42299999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50600000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53000000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59099999999999997}]}]},{"osd":6,"up_from":47,"seq":201863462930,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6108,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739044,"statfs":{"total":95995035648,"available":95988781056,"internally_reserved":0,"allocated":1134592,"data_stored":773594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 15 21:34:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.45000000000000001}]},{"osd":1,"last update":"Mon Apr 15 21:34:15 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.52500000000000002}]},{"osd":2,"last update":"Mon Apr 15 21:34:15 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.50800000000000001}]},{"osd":3,"last update":"Mon Apr 15 21:34:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.435},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.63900000000000001}]},{"osd":4,"last update":"Mon Apr 15 21:34:15 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.61299999999999999}]},{"osd":5,"last update":"Mon Apr 15 21:34:15 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.626},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.52100000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71599999999999997}]}]},{"osd":1,"up_from":14,"seq":60129542187,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6364,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738788,"statfs":{"total":95995035648,"available":95988518912,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 15 21:34:09 2024","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.32300000000000001,"15min":0.30399999999999999},"min":{"1min":0.23499999999999999,"5min":0.187,"15min":0.187},"max":{"1min":0.46300000000000002,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.38,"5min":0.33600000000000002,"15min":0.308},"min":{"1min":0.25700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.50900000000000001,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.32100000000000001}]},{"osd":2,"last update":"Mon Apr 15 21:33:36 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.33900000000000002,"15min":0.32700000000000001},"min":{"1min":0.29299999999999998,"5min":0.216,"15min":0.216},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.371},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.34699999999999998,"15min":0.33300000000000002},"min":{"1min":0.27800000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.35499999999999998}]},{"osd":3,"last update":"Mon Apr 15 21:34:05 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.42299999999999999,"15min":0.43099999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.47499999999999998,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.436,"5min":0.45400000000000001,"15min":0.45700000000000002},"min":{"1min":0.33500000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.58799999999999997,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.27900000000000003}]},{"osd":4,"last update":"Mon Apr 15 21:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.624}]},{"osd":5,"last update":"Mon Apr 15 21:33:51 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.64400000000000002}]},{"osd":6,"last update":"Mon Apr 15 21:34:12 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.501},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.60599999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52700000000000002}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Apr 15 21:34:10 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.307},"min":{"1min":0.224,"5min":0.182,"15min":0.182},"max":{"1min":0.48899999999999999,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.34599999999999997,"15min":0.32200000000000001},"min":{"1min":0.26800000000000002,"5min":0.191,"15min":0.191},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.39400000000000002}]},{"osd":2,"last update":"Mon Apr 15 21:33:40 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.38600000000000001,"15min":0.38400000000000001},"min":{"1min":0.30199999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.505,"5min":0.625,"15min":0.625},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.38200000000000001,"15min":0.377},"min":{"1min":0.29599999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.56100000000000005,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.36199999999999999}]},{"osd":3,"last update":"Mon Apr 15 21:34:06 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.441,"15min":0.44500000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.628,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.42999999999999999,"15min":0.433},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.40600000000000003}]},{"osd":4,"last update":"Mon Apr 15 21:33:27 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.64700000000000002}]},{"osd":5,"last update":"Mon Apr 15 21:33:48 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.61799999999999999}]},{"osd":6,"last update":"Mon Apr 15 21:34:13 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.60999999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67700000000000005}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6236,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738916,"statfs":{"total":95995035648,"available":95988649984,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 15 21:33:34 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.33400000000000002,"15min":0.32400000000000001},"min":{"1min":0.28399999999999997,"5min":0.19,"15min":0.19},"max":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.34100000000000003,"15min":0.32800000000000001},"min":{"1min":0.32700000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.311}]},{"osd":1,"last update":"Mon Apr 15 21:33:34 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.36299999999999999,"15min":0.34699999999999998},"min":{"1min":0.33300000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.35299999999999998,"15min":0.33500000000000002},"min":{"1min":0.28999999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.376}]},{"osd":3,"last update":"Mon Apr 15 21:34:07 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.442,"15min":0.438},"min":{"1min":0.29599999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.41499999999999998,"15min":0.40699999999999997},"min":{"1min":0.33400000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.36199999999999999}]},{"osd":4,"last update":"Mon Apr 15 21:33:28 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.55400000000000005}]},{"osd":5,"last update":"Mon Apr 15 21:33:48 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.495},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.56599999999999995}]},{"osd":6,"last update":"Mon Apr 15 21:34:14 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.47799999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54400000000000004}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 15 21:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.39800000000000002,"15min":0.39400000000000002},"min":{"1min":0.307,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.40200000000000002,"15min":0.40100000000000002},"min":{"1min":0.27900000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.28599999999999998}]},{"osd":1,"last update":"Mon Apr 15 21:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.42899999999999999,"15min":0.42699999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.40100000000000002,"15min":0.39700000000000002},"min":{"1min":0.26500000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.32000000000000001}]},{"osd":2,"last update":"Mon Apr 15 21:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.443,"15min":0.441},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.255},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"min":{"1min":0.29499999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.54600000000000004,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.372}]},{"osd":4,"last update":"Mon Apr 15 21:34:25 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.63,"15min":0.63300000000000001},"min":{"1min":0.439,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.88900000000000001,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.70599999999999996,"15min":0.72199999999999998},"min":{"1min":0.40699999999999997,"5min":0.379,"15min":0.379},"max":{"1min":0.83299999999999996,"5min":2.4060000000000001,"15min":2.4060000000000001},"last":0.73799999999999999}]},{"osd":5,"last update":"Mon Apr 15 21:33:48 2024","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.044,"15min":1.044},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":9.8650000000000002,"5min":9.8650000000000002,"15min":9.8650000000000002},"last":0.55900000000000005},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.0549999999999999,"15min":1.0549999999999999},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":9.8320000000000007,"5min":9.8320000000000007,"15min":9.8320000000000007},"last":0.72799999999999998}]},{"osd":6,"last update":"Mon Apr 15 21:34:14 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.55500000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57299999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 15 21:33:27 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.624}]},{"osd":1,"last update":"Mon Apr 15 21:33:27 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.56799999999999995}]},{"osd":2,"last update":"Mon Apr 15 21:33:27 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.58099999999999996}]},{"osd":3,"last update":"Mon Apr 15 21:33:27 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.70399999999999996}]},{"osd":5,"last update":"Mon Apr 15 21:33:48 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.37,"15min":0.37},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"last":0.434},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.496}]},{"osd":6,"last update":"Mon Apr 15 21:34:16 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.51500000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41799999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724567,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5660,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739492,"statfs":{"total":95995035648,"available":95989239808,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 15 21:33:49 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.64700000000000002}]},{"osd":1,"last update":"Mon Apr 15 21:33:49 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.60799999999999998}]},{"osd":2,"last update":"Mon Apr 15 21:33:49 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.59399999999999997}]},{"osd":3,"last update":"Mon Apr 15 21:33:49 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.56699999999999995}]},{"osd":4,"last update":"Mon Apr 15 21:33:49 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.49199999999999999}]},{"osd":6,"last update":"Mon Apr 15 21:34:16 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.47999999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53900000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-15T21:34:31.973 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph pg dump --format=json 2024-04-15T21:34:32.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:31 smithi139 ceph-mon[26919]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:32.189 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:34:32.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:32 smithi083 ceph-mon[24790]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-15T21:34:33.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:32 smithi139 ceph-mon[26919]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-15T21:34:33.911 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:34:33.912 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:33 smithi083 ceph-mon[24790]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:33.913 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2024-04-15T21:34:34.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:33 smithi139 ceph-mon[26919]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:34.567 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":199,"stamp":"2024-04-15T21:34:32.534763+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48748,"kb_used_data":6316,"kb_used_omap":0,"kb_used_meta":42368,"kb_avail":749912468,"statfs":{"total":767960285184,"available":767910367232,"internally_reserved":0,"allocated":6467584,"data_stored":3547758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43384832},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002229"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-04-15T21:33:40.554061+0000","last_change":"2024-04-15T21:33:40.554061+0000","last_active":"2024-04-15T21:33:40.554061+0000","last_peered":"2024-04-15T21:33:40.554061+0000","last_clean":"2024-04-15T21:33:40.554061+0000","last_became_active":"2024-04-15T21:33:40.553676+0000","last_became_peered":"2024-04-15T21:33:40.553676+0000","last_unstale":"2024-04-15T21:33:40.554061+0000","last_undegraded":"2024-04-15T21:33:40.554061+0000","last_fullsized":"2024-04-15T21:33:40.554061+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-15T21:31:34.794805+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-15T21:31:34.794805+0000","last_clean_scrub_stamp":"2024-04-15T21:31:34.794805+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-17T06:04:05.960190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2662400,"data_stored":2644640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67800000000000005}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.77800000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.745}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52500000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.755}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69599999999999995}]}]},{"osd":6,"up_from":47,"seq":201863462930,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6108,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739044,"statfs":{"total":95995035648,"available":95988781056,"internally_reserved":0,"allocated":1134592,"data_stored":773594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 15 21:34:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.45000000000000001}]},{"osd":1,"last update":"Mon Apr 15 21:34:15 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.52500000000000002}]},{"osd":2,"last update":"Mon Apr 15 21:34:15 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.50800000000000001}]},{"osd":3,"last update":"Mon Apr 15 21:34:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.435},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.63900000000000001}]},{"osd":4,"last update":"Mon Apr 15 21:34:15 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.61299999999999999}]},{"osd":5,"last update":"Mon Apr 15 21:34:15 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.626},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.52100000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71599999999999997}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6364,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738788,"statfs":{"total":95995035648,"available":95988518912,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 15 21:34:09 2024","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.32300000000000001,"15min":0.30399999999999999},"min":{"1min":0.23499999999999999,"5min":0.187,"15min":0.187},"max":{"1min":0.46300000000000002,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.247},{"interface":"front","average":{"1min":0.38,"5min":0.33600000000000002,"15min":0.308},"min":{"1min":0.25700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.50900000000000001,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.34899999999999998}]},{"osd":2,"last update":"Mon Apr 15 21:33:36 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.33900000000000002,"15min":0.32700000000000001},"min":{"1min":0.29299999999999998,"5min":0.216,"15min":0.216},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.34699999999999998,"15min":0.33300000000000002},"min":{"1min":0.27800000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.30499999999999999}]},{"osd":3,"last update":"Mon Apr 15 21:34:05 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.42299999999999999,"15min":0.43099999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.47499999999999998,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.38},{"interface":"front","average":{"1min":0.436,"5min":0.45400000000000001,"15min":0.45700000000000002},"min":{"1min":0.33500000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.58799999999999997,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.32800000000000001}]},{"osd":4,"last update":"Mon Apr 15 21:34:26 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.59799999999999998,"15min":0.59599999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.76600000000000001,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.62,"5min":0.60099999999999998,"15min":0.59799999999999998},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.63}]},{"osd":5,"last update":"Mon Apr 15 21:33:51 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.624}]},{"osd":6,"last update":"Mon Apr 15 21:34:12 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.58399999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65000000000000002}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Apr 15 21:34:10 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.307},"min":{"1min":0.224,"5min":0.182,"15min":0.182},"max":{"1min":0.48899999999999999,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.34599999999999997,"15min":0.32200000000000001},"min":{"1min":0.26800000000000002,"5min":0.191,"15min":0.191},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.27600000000000002}]},{"osd":2,"last update":"Mon Apr 15 21:33:40 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.38600000000000001,"15min":0.38400000000000001},"min":{"1min":0.30199999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.505,"5min":0.625,"15min":0.625},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.38200000000000001,"15min":0.377},"min":{"1min":0.29599999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.56100000000000005,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.26400000000000001}]},{"osd":3,"last update":"Mon Apr 15 21:34:06 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.441,"15min":0.44500000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.628,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.42999999999999999,"15min":0.433},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.314}]},{"osd":4,"last update":"Mon Apr 15 21:34:30 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.61799999999999999,"15min":0.61599999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.624,"5min":0.59899999999999998,"15min":0.59499999999999997},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.38700000000000001}]},{"osd":5,"last update":"Mon Apr 15 21:33:48 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.41699999999999998}]},{"osd":6,"last update":"Mon Apr 15 21:34:13 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.434},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.50800000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45000000000000001}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6236,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738916,"statfs":{"total":95995035648,"available":95988649984,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 15 21:33:34 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.33400000000000002,"15min":0.32400000000000001},"min":{"1min":0.28399999999999997,"5min":0.19,"15min":0.19},"max":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.34100000000000003,"15min":0.32800000000000001},"min":{"1min":0.32700000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.311}]},{"osd":1,"last update":"Mon Apr 15 21:33:34 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.36299999999999999,"15min":0.34699999999999998},"min":{"1min":0.33300000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.35299999999999998,"15min":0.33500000000000002},"min":{"1min":0.28999999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.376}]},{"osd":3,"last update":"Mon Apr 15 21:34:07 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.442,"15min":0.438},"min":{"1min":0.29599999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.41499999999999998,"15min":0.40699999999999997},"min":{"1min":0.33400000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.36199999999999999}]},{"osd":4,"last update":"Mon Apr 15 21:33:28 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.55400000000000005}]},{"osd":5,"last update":"Mon Apr 15 21:33:48 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.495},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.56599999999999995}]},{"osd":6,"last update":"Mon Apr 15 21:34:14 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.47799999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54400000000000004}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 15 21:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.39800000000000002,"15min":0.39400000000000002},"min":{"1min":0.307,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.40200000000000002,"15min":0.40100000000000002},"min":{"1min":0.27900000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.28599999999999998}]},{"osd":1,"last update":"Mon Apr 15 21:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.42899999999999999,"15min":0.42699999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.40100000000000002,"15min":0.39700000000000002},"min":{"1min":0.26500000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.32000000000000001}]},{"osd":2,"last update":"Mon Apr 15 21:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.443,"15min":0.441},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.255},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"min":{"1min":0.29499999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.54600000000000004,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.372}]},{"osd":4,"last update":"Mon Apr 15 21:34:25 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.63,"15min":0.63300000000000001},"min":{"1min":0.439,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.88900000000000001,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.70599999999999996,"15min":0.72199999999999998},"min":{"1min":0.40699999999999997,"5min":0.379,"15min":0.379},"max":{"1min":0.83299999999999996,"5min":2.4060000000000001,"15min":2.4060000000000001},"last":0.73799999999999999}]},{"osd":5,"last update":"Mon Apr 15 21:33:48 2024","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.044,"15min":1.044},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":9.8650000000000002,"5min":9.8650000000000002,"15min":9.8650000000000002},"last":0.55900000000000005},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.0549999999999999,"15min":1.0549999999999999},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":9.8320000000000007,"5min":9.8320000000000007,"15min":9.8320000000000007},"last":0.72799999999999998}]},{"osd":6,"last update":"Mon Apr 15 21:34:14 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.55500000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57299999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 15 21:33:27 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.624}]},{"osd":1,"last update":"Mon Apr 15 21:33:27 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.56799999999999995}]},{"osd":2,"last update":"Mon Apr 15 21:33:27 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.58099999999999996}]},{"osd":3,"last update":"Mon Apr 15 21:33:27 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.70399999999999996}]},{"osd":5,"last update":"Mon Apr 15 21:33:48 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.37,"15min":0.37},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"last":0.434},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.496}]},{"osd":6,"last update":"Mon Apr 15 21:34:16 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.51500000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41799999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724567,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5660,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739492,"statfs":{"total":95995035648,"available":95989239808,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 15 21:33:49 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.64700000000000002}]},{"osd":1,"last update":"Mon Apr 15 21:33:49 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.60799999999999998}]},{"osd":2,"last update":"Mon Apr 15 21:33:49 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.59399999999999997}]},{"osd":3,"last update":"Mon Apr 15 21:33:49 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.56699999999999995}]},{"osd":4,"last update":"Mon Apr 15 21:33:49 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.49199999999999999}]},{"osd":6,"last update":"Mon Apr 15 21:34:16 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.47999999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53900000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-15T21:34:34.568 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-15T21:34:34.568 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-15T21:34:34.568 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-15T21:34:34.569 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph health --format=json 2024-04-15T21:34:34.786 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/mon.smithi083/config 2024-04-15T21:34:35.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:35 smithi083 ceph-mon[24790]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-15T21:34:35.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:35 smithi083 ceph-mon[24790]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:36.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:35 smithi139 ceph-mon[26919]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-15T21:34:36.166 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:35 smithi139 ceph-mon[26919]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:36.520 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:34:36.520 INFO:teuthology.orchestra.run.smithi083.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-15T21:34:36.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:36 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/2892477780' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-15T21:34:37.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:36 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/2892477780' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-15T21:34:37.113 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-15T21:34:37.114 INFO:tasks.cephadm:Setup complete, yielding 2024-04-15T21:34:37.114 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-15T21:34:37.123 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-04-15T21:34:37.124 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- bash -c 'ceph orch status' 2024-04-15T21:34:38.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:37 smithi083 ceph-mon[24790]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:38.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:37 smithi139 ceph-mon[26919]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:39.111 INFO:teuthology.orchestra.run.smithi083.stdout:Backend: cephadm 2024-04-15T21:34:39.111 INFO:teuthology.orchestra.run.smithi083.stdout:Available: Yes 2024-04-15T21:34:39.111 INFO:teuthology.orchestra.run.smithi083.stdout:Paused: No 2024-04-15T21:34:39.777 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- bash -c 'ceph orch ps' 2024-04-15T21:34:39.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:39 smithi083 ceph-mon[24790]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:39.914 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:34:39.914 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:34:39.914 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:34:39.914 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:39 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:34:40.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:39 smithi139 ceph-mon[26919]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:40.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:34:40.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:34:40.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:34:40.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:39 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:34:41.044 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:40 smithi083 ceph-mon[24790]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:34:41.097 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:40 smithi139 ceph-mon[26919]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:34:41.712 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-15T21:34:41.712 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (4m) 2m ago 5m 17.6M - ba2b418f427c b36057f4bce8 2024-04-15T21:34:41.712 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (5m) 2m ago 5m 6405k - 17.2.0 e1d6a67b021e 016deb461f37 2024-04-15T21:34:41.712 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi139 smithi139 running (4m) 65s ago 4m 6405k - 17.2.0 e1d6a67b021e 3495d2d44d56 2024-04-15T21:34:41.712 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (3m) 2m ago 5m 45.5M - 8.3.5 dad864ee21e9 9218fa47bfde 2024-04-15T21:34:41.712 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.kxqarn smithi083 *:9283 running (6m) 2m ago 6m 448M - 17.2.0 e1d6a67b021e 9332afb3cbc0 2024-04-15T21:34:41.712 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi139.bqqykn smithi139 *:8443,9283 running (4m) 65s ago 4m 413M - 17.2.0 e1d6a67b021e af4b07488571 2024-04-15T21:34:41.712 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (6m) 2m ago 7m 44.5M 2048M 17.2.0 e1d6a67b021e 4e1e2c99fe19 2024-04-15T21:34:41.712 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi139 smithi139 running (4m) 65s ago 4m 39.4M 2048M 17.2.0 e1d6a67b021e 3e6950ee7383 2024-04-15T21:34:41.712 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (5m) 2m ago 5m 15.0M - 1dbe0e931976 17bc16074f49 2024-04-15T21:34:41.713 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi139 smithi139 *:9100 running (4m) 65s ago 4m 17.1M - 1dbe0e931976 df8a6fcc6bbd 2024-04-15T21:34:41.713 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4m) 2m ago 4m 36.6M 3216M 17.2.0 e1d6a67b021e a5cc4573e142 2024-04-15T21:34:41.713 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3m) 2m ago 3m 34.2M 3216M 17.2.0 e1d6a67b021e 4a973ccf3730 2024-04-15T21:34:41.713 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3m) 2m ago 3m 31.9M 3216M 17.2.0 e1d6a67b021e e1038d2e7462 2024-04-15T21:34:41.713 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2m) 2m ago 2m 28.0M 3216M 17.2.0 e1d6a67b021e aad5a54fb42f 2024-04-15T21:34:41.713 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi139 running (2m) 65s ago 2m 34.5M 3984M 17.2.0 e1d6a67b021e ca06e69e5c77 2024-04-15T21:34:41.713 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi139 running (119s) 65s ago 119s 32.2M 3984M 17.2.0 e1d6a67b021e 9696759277a4 2024-04-15T21:34:41.713 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi139 running (95s) 65s ago 95s 33.3M 3984M 17.2.0 e1d6a67b021e 8108f3669964 2024-04-15T21:34:41.713 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi139 running (71s) 65s ago 70s 10.1M 3984M 17.2.0 e1d6a67b021e ae9b789871b1 2024-04-15T21:34:41.713 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (4m) 2m ago 4m 53.8M - 514e6a882f6e 10a02d16091d 2024-04-15T21:34:41.713 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:41 smithi083 ceph-mon[24790]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:42.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:41 smithi139 ceph-mon[26919]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:42.317 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- bash -c 'ceph orch ls' 2024-04-15T21:34:42.946 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:42 smithi083 ceph-mon[24790]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:34:43.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:42 smithi139 ceph-mon[26919]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:34:43.897 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:43 smithi083 ceph-mon[24790]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:44.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:43 smithi139 ceph-mon[26919]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:44.290 INFO:teuthology.orchestra.run.smithi083.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-15T21:34:44.291 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-04-15T21:34:44.291 INFO:teuthology.orchestra.run.smithi083.stdout:crash 2/2 2m ago 6m * 2024-04-15T21:34:44.291 INFO:teuthology.orchestra.run.smithi083.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-04-15T21:34:44.291 INFO:teuthology.orchestra.run.smithi083.stdout:mgr 2/2 2m ago 6m count:2 2024-04-15T21:34:44.291 INFO:teuthology.orchestra.run.smithi083.stdout:mon 2/2 2m ago 5m smithi083:172.21.15.83=smithi083;smithi139:172.21.15.139=smithi139;count:2 2024-04-15T21:34:44.291 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-04-15T21:34:44.291 INFO:teuthology.orchestra.run.smithi083.stdout:osd 8 2m ago - 2024-04-15T21:34:44.291 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-04-15T21:34:44.914 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- bash -c 'ceph orch host ls' 2024-04-15T21:34:45.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:45 smithi083 ceph-mon[24790]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:34:45.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:45 smithi083 ceph-mon[24790]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:46.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:45 smithi139 ceph-mon[26919]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:34:46.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:45 smithi139 ceph-mon[26919]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:46.779 INFO:teuthology.orchestra.run.smithi083.stdout:HOST ADDR LABELS STATUS 2024-04-15T21:34:46.780 INFO:teuthology.orchestra.run.smithi083.stdout:smithi083 172.21.15.83 2024-04-15T21:34:46.780 INFO:teuthology.orchestra.run.smithi083.stdout:smithi139 172.21.15.139 2024-04-15T21:34:46.780 INFO:teuthology.orchestra.run.smithi083.stdout:2 hosts in cluster 2024-04-15T21:34:47.437 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- bash -c 'ceph orch device ls' 2024-04-15T21:34:47.972 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:47 smithi083 ceph-mon[24790]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:47.973 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:47 smithi083 ceph-mon[24790]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:34:48.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:47 smithi139 ceph-mon[26919]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:48.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:47 smithi139 ceph-mon[26919]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:34:49.393 INFO:teuthology.orchestra.run.smithi083.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-04-15T21:34:49.393 INFO:teuthology.orchestra.run.smithi083.stdout:smithi083 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT62040148400BGN 400G LVM detected, locked 2024-04-15T21:34:49.394 INFO:teuthology.orchestra.run.smithi083.stdout:smithi139 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620500N6400BGN 400G LVM detected, locked 2024-04-15T21:34:50.008 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:49 smithi083 ceph-mon[24790]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:50.031 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-15T21:34:50.040 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-04-15T21:34:50.040 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-04-15T21:34:50.072 INFO:teuthology.orchestra.run.smithi083.stderr:+ systemctl stop nfs-server 2024-04-15T21:34:50.081 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi139.front.sepia.ceph.com 2024-04-15T21:34:50.081 DEBUG:teuthology.orchestra.run.smithi139:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-04-15T21:34:50.107 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:49 smithi139 ceph-mon[26919]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:50.110 INFO:teuthology.orchestra.run.smithi139.stderr:+ systemctl stop nfs-server 2024-04-15T21:34:50.119 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-15T21:34:50.127 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-04-15T21:34:50.127 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-04-15T21:34:51.064 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:50 smithi139 ceph-mon[26919]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:34:51.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:50 smithi083 ceph-mon[24790]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:34:52.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:51 smithi083 ceph-mon[24790]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:52.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:51 smithi139 ceph-mon[26919]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:53.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:52 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-04-15T21:34:53.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:52 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-04-15T21:34:54.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:53 smithi083 ceph-mon[24790]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:34:54.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:53 smithi083 ceph-mon[24790]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:54.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:53 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-04-15T21:34:54.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:53 smithi083 ceph-mon[24790]: osdmap e60: 8 total, 8 up, 8 in 2024-04-15T21:34:54.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:53 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-04-15T21:34:54.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:53 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi083[24785]: 2024-04-15T21:34:53.705+0000 7f03c5a6f700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-15T21:34:54.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:53 smithi139 ceph-mon[26919]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:34:54.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:53 smithi139 ceph-mon[26919]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:54.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:53 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-04-15T21:34:54.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:53 smithi139 ceph-mon[26919]: osdmap e60: 8 total, 8 up, 8 in 2024-04-15T21:34:54.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:53 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-04-15T21:34:54.395 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-04-15T21:34:54.403 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-04-15T21:34:54.403 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph orch ls -f json 2024-04-15T21:34:55.000 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:54 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-04-15T21:34:55.000 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:54 smithi083 ceph-mon[24790]: osdmap e61: 8 total, 8 up, 8 in 2024-04-15T21:34:55.001 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:54 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-04-15T21:34:55.001 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:54 smithi083 ceph-mon[24790]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-15T21:34:55.001 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:54 smithi083 ceph-mon[24790]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-15T21:34:55.001 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:54 smithi083 ceph-mon[24790]: osdmap e62: 8 total, 8 up, 8 in 2024-04-15T21:34:55.001 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:54 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-04-15T21:34:55.001 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:54 smithi083 ceph-mon[24790]: fsmap foofs:0 2024-04-15T21:34:55.001 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:54 smithi083 ceph-mon[24790]: Saving service mds.foofs spec with placement count:2 2024-04-15T21:34:55.001 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:54 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:34:55.001 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:54 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:34:55.001 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:54 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:34:55.001 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:54 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:34:55.001 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:54 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:34:55.001 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:54 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:34:55.002 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:54 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.lgyynu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-15T21:34:55.002 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:54 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.lgyynu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-15T21:34:55.002 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:54 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:34:55.002 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:54 smithi083 ceph-mon[24790]: Deploying daemon mds.foofs.smithi083.lgyynu on smithi083 2024-04-15T21:34:55.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:54 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-04-15T21:34:55.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:54 smithi139 ceph-mon[26919]: osdmap e61: 8 total, 8 up, 8 in 2024-04-15T21:34:55.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:54 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-04-15T21:34:55.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:54 smithi139 ceph-mon[26919]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-15T21:34:55.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:54 smithi139 ceph-mon[26919]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-15T21:34:55.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:54 smithi139 ceph-mon[26919]: osdmap e62: 8 total, 8 up, 8 in 2024-04-15T21:34:55.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:54 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-04-15T21:34:55.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:54 smithi139 ceph-mon[26919]: fsmap foofs:0 2024-04-15T21:34:55.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:54 smithi139 ceph-mon[26919]: Saving service mds.foofs spec with placement count:2 2024-04-15T21:34:55.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:54 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:34:55.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:54 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:34:55.114 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:54 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:34:55.114 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:54 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:34:55.114 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:54 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:34:55.114 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:54 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:34:55.114 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:54 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.lgyynu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-15T21:34:55.114 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:54 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.lgyynu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-15T21:34:55.114 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:54 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:34:55.114 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:54 smithi139 ceph-mon[26919]: Deploying daemon mds.foofs.smithi083.lgyynu on smithi083 2024-04-15T21:34:56.069 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:55 smithi083 ceph-mon[24790]: pgmap v213: 65 pgs: 9 creating+peering, 2 active+clean, 54 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:56.069 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:55 smithi083 ceph-mon[24790]: osdmap e63: 8 total, 8 up, 8 in 2024-04-15T21:34:56.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:55 smithi139 ceph-mon[26919]: pgmap v213: 65 pgs: 9 creating+peering, 2 active+clean, 54 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:56.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:55 smithi139 ceph-mon[26919]: osdmap e63: 8 total, 8 up, 8 in 2024-04-15T21:34:56.894 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:56 smithi083 ceph-mon[24790]: osdmap e64: 8 total, 8 up, 8 in 2024-04-15T21:34:57.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:56 smithi139 ceph-mon[26919]: osdmap e64: 8 total, 8 up, 8 in 2024-04-15T21:34:57.574 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:34:57.574 INFO:teuthology.orchestra.run.smithi083.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-15T21:28:26.307361Z", "last_refresh": "2024-04-15T21:32:09.414233Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-15T21:28:20.426550Z", "last_refresh": "2024-04-15T21:32:09.414297Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-15T21:28:23.509396Z", "last_refresh": "2024-04-15T21:32:09.414357Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-15T21:34:53.724354Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-15T21:34:53.719504Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-15T21:28:19.278773Z", "last_refresh": "2024-04-15T21:32:09.414163Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi083:172.21.15.83=smithi083", "smithi139:172.21.15.139=smithi139"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-15T21:29:12.194761Z", "last_refresh": "2024-04-15T21:32:09.414003Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-15T21:28:25.127434Z", "last_refresh": "2024-04-15T21:32:09.414416Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-15T21:32:09.414532Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-15T21:28:22.313633Z", "last_refresh": "2024-04-15T21:32:09.414474Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-15T21:34:57.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:57 smithi083 ceph-mon[24790]: pgmap v216: 65 pgs: 9 creating+peering, 23 active+clean, 33 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:58.100 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:57 smithi139 ceph-mon[26919]: pgmap v216: 65 pgs: 9 creating+peering, 23 active+clean, 33 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:58.555 INFO:tasks.cephadm:mds.foofs has 0/2 2024-04-15T21:34:58.745 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:58 smithi139 ceph-mon[26919]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-15T21:34:58.745 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:58 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:34:58.745 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:58 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi139.akqukn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-15T21:34:58.745 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:58 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi139.akqukn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-15T21:34:58.745 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:58 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:34:58.745 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:58 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:34:58.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:58 smithi083 ceph-mon[24790]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-15T21:34:58.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:58 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:34:58.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:58 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi139.akqukn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-15T21:34:58.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:58 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi139.akqukn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-15T21:34:58.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:58 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:34:58.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:58 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:34:59.556 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph orch ls -f json 2024-04-15T21:34:59.796 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:59 smithi139 ceph-mon[26919]: Deploying daemon mds.foofs.smithi139.akqukn on smithi139 2024-04-15T21:34:59.797 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:59 smithi139 ceph-mon[26919]: pgmap v217: 65 pgs: 4 creating+peering, 61 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:59.797 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:59 smithi139 ceph-mon[26919]: mds.? [v2:172.21.15.83:6834/3077026664,v1:172.21.15.83:6835/3077026664] up:boot 2024-04-15T21:34:59.797 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:59 smithi139 ceph-mon[26919]: daemon mds.foofs.smithi083.lgyynu assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-04-15T21:34:59.797 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:59 smithi139 ceph-mon[26919]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-15T21:34:59.797 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:59 smithi139 ceph-mon[26919]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-15T21:34:59.797 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:59 smithi139 ceph-mon[26919]: Cluster is now healthy 2024-04-15T21:34:59.797 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:59 smithi139 ceph-mon[26919]: fsmap foofs:0 1 up:standby 2024-04-15T21:34:59.797 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:59 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi083.lgyynu"}]: dispatch 2024-04-15T21:34:59.797 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:59 smithi139 ceph-mon[26919]: fsmap foofs:1 {0=foofs.smithi083.lgyynu=up:creating} 2024-04-15T21:34:59.797 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:34:59 smithi139 ceph-mon[26919]: daemon mds.foofs.smithi083.lgyynu is now active in filesystem foofs as rank 0 2024-04-15T21:34:59.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:59 smithi083 ceph-mon[24790]: Deploying daemon mds.foofs.smithi139.akqukn on smithi139 2024-04-15T21:34:59.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:59 smithi083 ceph-mon[24790]: pgmap v217: 65 pgs: 4 creating+peering, 61 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:34:59.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:59 smithi083 ceph-mon[24790]: mds.? [v2:172.21.15.83:6834/3077026664,v1:172.21.15.83:6835/3077026664] up:boot 2024-04-15T21:34:59.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:59 smithi083 ceph-mon[24790]: daemon mds.foofs.smithi083.lgyynu assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-04-15T21:34:59.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:59 smithi083 ceph-mon[24790]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-15T21:34:59.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:59 smithi083 ceph-mon[24790]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-15T21:34:59.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:59 smithi083 ceph-mon[24790]: Cluster is now healthy 2024-04-15T21:34:59.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:59 smithi083 ceph-mon[24790]: fsmap foofs:0 1 up:standby 2024-04-15T21:34:59.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:59 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi083.lgyynu"}]: dispatch 2024-04-15T21:34:59.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:59 smithi083 ceph-mon[24790]: fsmap foofs:1 {0=foofs.smithi083.lgyynu=up:creating} 2024-04-15T21:34:59.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:34:59 smithi083 ceph-mon[24790]: daemon mds.foofs.smithi083.lgyynu is now active in filesystem foofs as rank 0 2024-04-15T21:35:00.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:00 smithi083 ceph-mon[24790]: mds.? [v2:172.21.15.83:6834/3077026664,v1:172.21.15.83:6835/3077026664] up:active 2024-04-15T21:35:00.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:00 smithi083 ceph-mon[24790]: fsmap foofs:1 {0=foofs.smithi083.lgyynu=up:active} 2024-04-15T21:35:00.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:00 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:00.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:00 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:35:00.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:00 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:00.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:00 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:35:01.096 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:00 smithi139 ceph-mon[26919]: mds.? [v2:172.21.15.83:6834/3077026664,v1:172.21.15.83:6835/3077026664] up:active 2024-04-15T21:35:01.096 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:00 smithi139 ceph-mon[26919]: fsmap foofs:1 {0=foofs.smithi083.lgyynu=up:active} 2024-04-15T21:35:01.097 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:00 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:01.097 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:00 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:35:01.097 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:00 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:01.097 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:00 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:35:01.591 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:35:01.591 INFO:teuthology.orchestra.run.smithi083.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-15T21:28:26.307361Z", "last_refresh": "2024-04-15T21:32:09.414233Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-15T21:28:20.426550Z", "last_refresh": "2024-04-15T21:32:09.414297Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-15T21:28:23.509396Z", "last_refresh": "2024-04-15T21:32:09.414357Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-15T21:34:53.724354Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-15T21:34:53.719504Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-15T21:28:19.278773Z", "last_refresh": "2024-04-15T21:32:09.414163Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi083:172.21.15.83=smithi083", "smithi139:172.21.15.139=smithi139"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-15T21:29:12.194761Z", "last_refresh": "2024-04-15T21:32:09.414003Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-15T21:28:25.127434Z", "last_refresh": "2024-04-15T21:32:09.414416Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-15T21:32:09.414532Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-15T21:28:22.313633Z", "last_refresh": "2024-04-15T21:32:09.414474Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-15T21:35:01.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:01 smithi083 ceph-mon[24790]: pgmap v218: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 149 B/s wr, 2 op/s 2024-04-15T21:35:01.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:01 smithi083 ceph-mon[24790]: mds.? [v2:172.21.15.139:6832/2626643012,v1:172.21.15.139:6833/2626643012] up:boot 2024-04-15T21:35:01.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:01 smithi083 ceph-mon[24790]: fsmap foofs:1 {0=foofs.smithi083.lgyynu=up:active} 1 up:standby 2024-04-15T21:35:01.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:01 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi139.akqukn"}]: dispatch 2024-04-15T21:35:01.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:01 smithi083 ceph-mon[24790]: fsmap foofs:1 {0=foofs.smithi083.lgyynu=up:active} 1 up:standby 2024-04-15T21:35:02.098 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:01 smithi139 ceph-mon[26919]: pgmap v218: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 149 B/s wr, 2 op/s 2024-04-15T21:35:02.098 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:01 smithi139 ceph-mon[26919]: mds.? [v2:172.21.15.139:6832/2626643012,v1:172.21.15.139:6833/2626643012] up:boot 2024-04-15T21:35:02.099 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:01 smithi139 ceph-mon[26919]: fsmap foofs:1 {0=foofs.smithi083.lgyynu=up:active} 1 up:standby 2024-04-15T21:35:02.099 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:01 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi139.akqukn"}]: dispatch 2024-04-15T21:35:02.099 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:01 smithi139 ceph-mon[26919]: fsmap foofs:1 {0=foofs.smithi083.lgyynu=up:active} 1 up:standby 2024-04-15T21:35:02.276 INFO:tasks.cephadm:mds.foofs has 0/2 2024-04-15T21:35:03.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:02 smithi083 ceph-mon[24790]: from='client.14500 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-15T21:35:03.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:02 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:03.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:02 smithi139 ceph-mon[26919]: from='client.14500 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-15T21:35:03.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:02 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:03.277 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph orch ls -f json 2024-04-15T21:35:03.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:03 smithi083 ceph-mon[24790]: pgmap v219: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-04-15T21:35:03.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:03 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:04.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:03 smithi139 ceph-mon[26919]: pgmap v219: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-04-15T21:35:04.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:03 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:06.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:05 smithi083 ceph-mon[24790]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-15T21:35:06.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:05 smithi139 ceph-mon[26919]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-15T21:35:07.268 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:35:07.269 INFO:teuthology.orchestra.run.smithi083.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-15T21:28:26.307361Z", "last_refresh": "2024-04-15T21:32:09.414233Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-15T21:28:20.426550Z", "last_refresh": "2024-04-15T21:32:09.414297Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-15T21:28:23.509396Z", "last_refresh": "2024-04-15T21:32:09.414357Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-15T21:34:53.724354Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-15T21:34:53.719504Z", "last_refresh": "2024-04-15T21:35:02.659202Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-15T21:28:19.278773Z", "last_refresh": "2024-04-15T21:32:09.414163Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi083:172.21.15.83=smithi083", "smithi139:172.21.15.139=smithi139"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-15T21:29:12.194761Z", "last_refresh": "2024-04-15T21:32:09.414003Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-15T21:28:25.127434Z", "last_refresh": "2024-04-15T21:32:09.414416Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-15T21:32:09.414532Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-15T21:28:22.313633Z", "last_refresh": "2024-04-15T21:32:09.414474Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-15T21:35:07.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:07 smithi083 ceph-mon[24790]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-04-15T21:35:07.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:07 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:07.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:07 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:07.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:07 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:35:07.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:07 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:07.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:07 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:35:07.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:07 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:08.046 INFO:tasks.cephadm:mds.foofs has 1/2 2024-04-15T21:35:08.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:07 smithi139 ceph-mon[26919]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-04-15T21:35:08.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:07 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:08.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:07 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:08.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:07 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:35:08.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:07 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:08.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:07 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:35:08.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:07 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:09.048 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- ceph orch ls -f json 2024-04-15T21:35:09.079 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:08 smithi083 ceph-mon[24790]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-15T21:35:09.080 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:08 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:09.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:08 smithi139 ceph-mon[26919]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-15T21:35:09.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:08 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:10.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:09 smithi083 ceph-mon[24790]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-15T21:35:10.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:09 smithi139 ceph-mon[26919]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-15T21:35:11.095 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:35:11.095 INFO:teuthology.orchestra.run.smithi083.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-15T21:28:26.307361Z", "last_refresh": "2024-04-15T21:35:07.297954Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-15T21:28:20.426550Z", "last_refresh": "2024-04-15T21:35:02.658569Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-15T21:28:23.509396Z", "last_refresh": "2024-04-15T21:35:07.298188Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-15T21:34:53.724354Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-15T21:34:53.719504Z", "last_refresh": "2024-04-15T21:35:02.659202Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-15T21:28:19.278773Z", "last_refresh": "2024-04-15T21:35:02.658711Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi083:172.21.15.83=smithi083", "smithi139:172.21.15.139=smithi139"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-15T21:29:12.194761Z", "last_refresh": "2024-04-15T21:35:02.658785Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-15T21:28:25.127434Z", "last_refresh": "2024-04-15T21:35:02.658866Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-15T21:35:02.658933Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-15T21:28:22.313633Z", "last_refresh": "2024-04-15T21:35:07.298408Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-15T21:35:11.787 INFO:tasks.cephadm:mds.foofs has 2/2 2024-04-15T21:35:11.787 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-15T21:35:11.797 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-04-15T21:35:11.797 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-04-15T21:35:12.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:11 smithi083 ceph-mon[24790]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-15T21:35:12.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:11 smithi139 ceph-mon[26919]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-15T21:35:13.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:12 smithi083 ceph-mon[24790]: from='client.14508 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-15T21:35:13.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:12 smithi139 ceph-mon[26919]: from='client.14508 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-15T21:35:14.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:13 smithi083 ceph-mon[24790]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 2 op/s 2024-04-15T21:35:14.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:13 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:14.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:13 smithi139 ceph-mon[26919]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 2 op/s 2024-04-15T21:35:14.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:13 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:15.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:14 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-04-15T21:35:15.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:14 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-04-15T21:35:15.808 INFO:teuthology.orchestra.run.smithi083.stdout:NFS Cluster Created Successfully 2024-04-15T21:35:16.053 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:15 smithi083 ceph-mon[24790]: from='client.14512 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:16.053 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:15 smithi083 ceph-mon[24790]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-15T21:35:16.053 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:15 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-04-15T21:35:16.053 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:15 smithi083 ceph-mon[24790]: osdmap e65: 8 total, 8 up, 8 in 2024-04-15T21:35:16.053 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:15 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-15T21:35:16.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:15 smithi139 ceph-mon[26919]: from='client.14512 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:16.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:15 smithi139 ceph-mon[26919]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-15T21:35:16.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:15 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-04-15T21:35:16.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:15 smithi139 ceph-mon[26919]: osdmap e65: 8 total, 8 up, 8 in 2024-04-15T21:35:16.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:15 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-15T21:35:16.477 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-04-15T21:35:17.041 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-04-15T21:35:17.041 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:16 smithi083 ceph-mon[24790]: osdmap e66: 8 total, 8 up, 8 in 2024-04-15T21:35:17.041 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:17.041 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:35:17.041 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:17.041 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:35:17.041 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:17.041 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:17.041 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.vuiewy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-15T21:35:17.041 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.vuiewy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-15T21:35:17.041 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-15T21:35:17.042 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-15T21:35:17.042 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:17.042 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-15T21:35:17.042 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-15T21:35:17.042 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.vuiewy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-15T21:35:17.042 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.vuiewy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-15T21:35:17.042 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:16 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:17.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-04-15T21:35:17.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:16 smithi139 ceph-mon[26919]: osdmap e66: 8 total, 8 up, 8 in 2024-04-15T21:35:17.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:17.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:35:17.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:17.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:35:17.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:17.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:17.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.vuiewy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-15T21:35:17.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.vuiewy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-15T21:35:17.114 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-15T21:35:17.114 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-15T21:35:17.114 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:17.114 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-15T21:35:17.114 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-15T21:35:17.114 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.vuiewy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-15T21:35:17.114 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.vuiewy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-15T21:35:17.114 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:16 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:18.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:17 smithi083 ceph-mon[24790]: Saving service nfs.foo spec with placement count:2 2024-04-15T21:35:18.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:17 smithi083 ceph-mon[24790]: Creating key for client.nfs.foo.0.0.smithi083.vuiewy 2024-04-15T21:35:18.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:17 smithi083 ceph-mon[24790]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-15T21:35:18.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:17 smithi083 ceph-mon[24790]: Rados config object exists: conf-nfs.foo 2024-04-15T21:35:18.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:17 smithi083 ceph-mon[24790]: Creating key for client.nfs.foo.0.0.smithi083.vuiewy-rgw 2024-04-15T21:35:18.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:17 smithi083 ceph-mon[24790]: Deploying daemon nfs.foo.0.0.smithi083.vuiewy on smithi083 2024-04-15T21:35:18.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:17 smithi083 ceph-mon[24790]: pgmap v228: 97 pgs: 22 unknown, 75 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:35:18.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:17 smithi083 ceph-mon[24790]: osdmap e67: 8 total, 8 up, 8 in 2024-04-15T21:35:18.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:17 smithi139 ceph-mon[26919]: Saving service nfs.foo spec with placement count:2 2024-04-15T21:35:18.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:17 smithi139 ceph-mon[26919]: Creating key for client.nfs.foo.0.0.smithi083.vuiewy 2024-04-15T21:35:18.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:17 smithi139 ceph-mon[26919]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-15T21:35:18.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:17 smithi139 ceph-mon[26919]: Rados config object exists: conf-nfs.foo 2024-04-15T21:35:18.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:17 smithi139 ceph-mon[26919]: Creating key for client.nfs.foo.0.0.smithi083.vuiewy-rgw 2024-04-15T21:35:18.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:17 smithi139 ceph-mon[26919]: Deploying daemon nfs.foo.0.0.smithi083.vuiewy on smithi083 2024-04-15T21:35:18.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:17 smithi139 ceph-mon[26919]: pgmap v228: 97 pgs: 22 unknown, 75 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-15T21:35:18.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:17 smithi139 ceph-mon[26919]: osdmap e67: 8 total, 8 up, 8 in 2024-04-15T21:35:19.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:19 smithi083 ceph-mon[24790]: pgmap v230: 97 pgs: 10 unknown, 87 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-15T21:35:19.761 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:19 smithi083 ceph-mon[24790]: mgrmap e20: smithi083.kxqarn(active, since 5m), standbys: smithi139.bqqykn 2024-04-15T21:35:19.803 INFO:teuthology.orchestra.run.smithi083.stderr:Invalid command: Unexpected argument '--clusterid' 2024-04-15T21:35:19.803 INFO:teuthology.orchestra.run.smithi083.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-04-15T21:35:19.803 INFO:teuthology.orchestra.run.smithi083.stderr:Error EINVAL: invalid command 2024-04-15T21:35:19.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:19 smithi139 ceph-mon[26919]: pgmap v230: 97 pgs: 10 unknown, 87 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-15T21:35:19.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:19 smithi139 ceph-mon[26919]: mgrmap e20: smithi083.kxqarn(active, since 5m), standbys: smithi139.bqqykn 2024-04-15T21:35:20.154 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-04-15T21:35:20.154 INFO:teuthology.orchestra.run.smithi083.stdout: "bind": "/fake", 2024-04-15T21:35:20.154 INFO:teuthology.orchestra.run.smithi083.stdout: "fs": "foofs", 2024-04-15T21:35:20.154 INFO:teuthology.orchestra.run.smithi083.stdout: "path": "/", 2024-04-15T21:35:20.154 INFO:teuthology.orchestra.run.smithi083.stdout: "cluster": "foo", 2024-04-15T21:35:20.154 INFO:teuthology.orchestra.run.smithi083.stdout: "mode": "RW" 2024-04-15T21:35:20.154 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-04-15T21:35:20.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:20 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-15T21:35:20.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:20 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-15T21:35:20.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:20 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:20.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:20 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi139.nuvjad", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-15T21:35:20.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:20 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi139.nuvjad", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-15T21:35:20.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:20 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-15T21:35:20.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:20 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-15T21:35:20.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:20 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:20.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:20 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-15T21:35:20.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:20 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-15T21:35:20.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:20 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:20.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:20 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi139.nuvjad", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-15T21:35:20.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:20 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi139.nuvjad", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-15T21:35:20.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:20 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-15T21:35:20.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:20 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-15T21:35:20.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:20 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:20.904 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-04-15T21:35:21.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:21 smithi083 ceph-mon[24790]: from='client.14534 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:21.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:21 smithi083 ceph-mon[24790]: Creating key for client.nfs.foo.1.0.smithi139.nuvjad 2024-04-15T21:35:21.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:21 smithi083 ceph-mon[24790]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-15T21:35:21.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:21 smithi083 ceph-mon[24790]: pgmap v231: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-15T21:35:21.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:21 smithi139 ceph-mon[26919]: from='client.14534 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:21.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:21 smithi139 ceph-mon[26919]: Creating key for client.nfs.foo.1.0.smithi139.nuvjad 2024-04-15T21:35:21.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:21 smithi139 ceph-mon[26919]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-15T21:35:21.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:21 smithi139 ceph-mon[26919]: pgmap v231: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-15T21:35:23.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:23 smithi139 ceph-mon[26919]: pgmap v232: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 131 B/s rd, 263 B/s wr, 0 op/s 2024-04-15T21:35:23.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:23 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-15T21:35:23.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:23 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-15T21:35:23.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:23 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi139.nuvjad-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-15T21:35:23.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:23 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi139.nuvjad-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-15T21:35:23.845 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:23 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:23.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:23 smithi083 ceph-mon[24790]: pgmap v232: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 131 B/s rd, 263 B/s wr, 0 op/s 2024-04-15T21:35:23.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:23 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-15T21:35:23.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:23 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-15T21:35:23.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:23 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi139.nuvjad-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-15T21:35:23.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:23 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi139.nuvjad-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-15T21:35:23.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:23 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:24.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:24 smithi083 ceph-mon[24790]: from='client.14550 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:24.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:24 smithi083 ceph-mon[24790]: Rados config object exists: conf-nfs.foo 2024-04-15T21:35:24.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:24 smithi083 ceph-mon[24790]: Creating key for client.nfs.foo.1.0.smithi139.nuvjad-rgw 2024-04-15T21:35:24.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:24 smithi083 ceph-mon[24790]: Deploying daemon nfs.foo.1.0.smithi139.nuvjad on smithi139 2024-04-15T21:35:24.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:24 smithi139 ceph-mon[26919]: from='client.14550 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:24.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:24 smithi139 ceph-mon[26919]: Rados config object exists: conf-nfs.foo 2024-04-15T21:35:24.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:24 smithi139 ceph-mon[26919]: Creating key for client.nfs.foo.1.0.smithi139.nuvjad-rgw 2024-04-15T21:35:24.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:24 smithi139 ceph-mon[26919]: Deploying daemon nfs.foo.1.0.smithi139.nuvjad on smithi139 2024-04-15T21:35:25.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:25 smithi139 ceph-mon[26919]: from='client.14564 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:25.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:25 smithi139 ceph-mon[26919]: pgmap v233: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-04-15T21:35:25.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:25 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:25.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:25 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:35:25.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:25 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:25.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:25 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:35:25.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:25 smithi083 ceph-mon[24790]: from='client.14564 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:25.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:25 smithi083 ceph-mon[24790]: pgmap v233: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-04-15T21:35:25.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:25 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:25.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:25 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:35:25.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:25 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:25.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:25 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:35:26.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:26 smithi083 ceph-mon[24790]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:26.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:26 smithi139 ceph-mon[26919]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:27.806 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:27 smithi083 ceph-mon[24790]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-04-15T21:35:27.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:27 smithi139 ceph-mon[26919]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-04-15T21:35:28.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:28 smithi083 ceph-mon[24790]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:28.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:28.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 4]}]: dispatch 2024-04-15T21:35:28.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-04-15T21:35:28.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-04-15T21:35:28.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-15T21:35:28.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:35:28.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:35:28.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:28 smithi139 ceph-mon[26919]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:28.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:28.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 4]}]: dispatch 2024-04-15T21:35:28.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-04-15T21:35:28.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-04-15T21:35:28.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-15T21:35:28.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:35:28.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:35:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:29 smithi083 ceph-mon[24790]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:29 smithi083 ceph-mon[24790]: pgmap v235: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.3 KiB/s wr, 6 op/s 2024-04-15T21:35:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 4]}]': finished 2024-04-15T21:35:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-04-15T21:35:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-04-15T21:35:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:29 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-04-15T21:35:29.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:29 smithi083 ceph-mon[24790]: osdmap e68: 8 total, 8 up, 8 in 2024-04-15T21:35:29.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:29 smithi139 ceph-mon[26919]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:29.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:29 smithi139 ceph-mon[26919]: pgmap v235: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.3 KiB/s wr, 6 op/s 2024-04-15T21:35:29.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:29 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:29.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:29 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 4]}]': finished 2024-04-15T21:35:29.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:29 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-04-15T21:35:29.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:29 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-04-15T21:35:29.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:29 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-04-15T21:35:29.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:29 smithi139 ceph-mon[26919]: osdmap e68: 8 total, 8 up, 8 in 2024-04-15T21:35:30.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:30 smithi083 ceph-mon[24790]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:30.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:30 smithi083 ceph-mon[24790]: osdmap e69: 8 total, 8 up, 8 in 2024-04-15T21:35:30.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:30 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:30.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:30 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:30.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:30 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:30.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:30 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:35:30.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:30 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:30.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:30 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:35:30.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:30 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:30.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:30 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:30.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:30 smithi139 ceph-mon[26919]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:30.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:30 smithi139 ceph-mon[26919]: osdmap e69: 8 total, 8 up, 8 in 2024-04-15T21:35:30.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:30 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:30.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:30 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:30.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:30 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:30.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:30 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:35:30.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:30 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:30.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:30 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:35:30.864 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:30 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:30.864 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:30 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:30.898 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo ?:2049 2/2 2s ago 15s count:2 2024-04-15T21:35:31.761 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-15T21:35:31.772 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-04-15T21:35:31.772 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-04-15T21:35:31.802 INFO:teuthology.orchestra.run.smithi083.stderr:+ mkdir /mnt/foo 2024-04-15T21:35:31.811 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-04-15T21:35:31.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:31 smithi083 ceph-mon[24790]: pgmap v238: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s 2024-04-15T21:35:31.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:31 smithi139 ceph-mon[26919]: pgmap v238: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s 2024-04-15T21:35:31.877 INFO:teuthology.orchestra.run.smithi083.stderr:++ hostname 2024-04-15T21:35:31.878 INFO:teuthology.orchestra.run.smithi083.stderr:+ mount -t nfs smithi083:/fake /mnt/foo -o sync 2024-04-15T21:35:32.299 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-04-15T21:35:32.368 INFO:teuthology.orchestra.run.smithi083.stderr:+ echo test 2024-04-15T21:35:32.379 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-04-15T21:35:32.446 INFO:teuthology.orchestra.run.smithi083.stderr:+ sync 2024-04-15T21:35:32.655 INFO:teuthology.run_tasks:Running task parallel... 2024-04-15T21:35:32.667 INFO:teuthology.task.parallel:starting parallel... 2024-04-15T21:35:32.667 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-15T21:35:32.668 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-15T21:35:32.668 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-04-15T21:35:32.669 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -e sha1=a9a752df26c63acad72e1b3569fd79a515ca0765 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-15T21:35:32.669 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-15T21:35:32.669 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-04-15T21:35:32.671 INFO:teuthology.task.exec:Executing custom commands... 2024-04-15T21:35:32.671 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-04-15T21:35:32.671 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-04-15T21:35:32.781 INFO:teuthology.orchestra.run.smithi083.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-04-15T21:35:32.782 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-15T21:35:32.782 INFO:teuthology.orchestra.run.smithi083.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-04-15T21:35:32.783 INFO:teuthology.orchestra.run.smithi083.stdout:3 of 5 processes prepared for launch 0 sec 2024-04-15T21:35:32.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:32 smithi083 ceph-mon[24790]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:32.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:32 smithi139 ceph-mon[26919]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:32.883 INFO:teuthology.orchestra.run.smithi083.stdout:5 of 5 processes prepared for launch 0 sec 2024-04-15T21:35:32.883 INFO:teuthology.orchestra.run.smithi083.stdout:releasing clients 2024-04-15T21:35:33.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:33 smithi083 ceph-mon[24790]: pgmap v239: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-04-15T21:35:33.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:33 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:33.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:33 smithi139 ceph-mon[26919]: pgmap v239: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-04-15T21:35:33.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:33 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:33.883 INFO:teuthology.orchestra.run.smithi083.stdout: 5 116 26.80 MB/sec warmup 1 sec latency 40.472 ms 2024-04-15T21:35:34.883 INFO:teuthology.orchestra.run.smithi083.stdout: 5 248 23.48 MB/sec warmup 2 sec latency 31.463 ms 2024-04-15T21:35:35.742 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:35 smithi083 ceph-mon[24790]: pgmap v240: 97 pgs: 97 active+clean; 4.1 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 454 KiB/s wr, 15 op/s; 0 B/s, 0 objects/s recovering 2024-04-15T21:35:35.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:35 smithi139 ceph-mon[26919]: pgmap v240: 97 pgs: 97 active+clean; 4.1 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 454 KiB/s wr, 15 op/s; 0 B/s, 0 objects/s recovering 2024-04-15T21:35:35.883 INFO:teuthology.orchestra.run.smithi083.stdout: 5 362 21.55 MB/sec warmup 3 sec latency 33.124 ms 2024-04-15T21:35:35.984 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -e sha1=a9a752df26c63acad72e1b3569fd79a515ca0765 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-15T21:35:36.883 INFO:teuthology.orchestra.run.smithi083.stdout: 5 479 20.94 MB/sec warmup 4 sec latency 32.059 ms 2024-04-15T21:35:37.837 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:37 smithi083 ceph-mon[24790]: pgmap v241: 97 pgs: 97 active+clean; 23 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 2.9 MiB/s wr, 55 op/s; 0 B/s, 0 objects/s recovering 2024-04-15T21:35:37.883 INFO:teuthology.orchestra.run.smithi083.stdout: 5 628 20.79 MB/sec warmup 5 sec latency 47.027 ms 2024-04-15T21:35:38.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:37 smithi139 ceph-mon[26919]: pgmap v241: 97 pgs: 97 active+clean; 23 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 2.9 MiB/s wr, 55 op/s; 0 B/s, 0 objects/s recovering 2024-04-15T21:35:38.883 INFO:teuthology.orchestra.run.smithi083.stdout: 5 1043 18.80 MB/sec warmup 6 sec latency 24.044 ms 2024-04-15T21:35:39.623 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:39 smithi083 ceph-mon[24790]: pgmap v242: 97 pgs: 97 active+clean; 81 MiB data, 124 MiB used, 715 GiB / 715 GiB avail; 220 B/s rd, 8.9 MiB/s wr, 385 op/s; 0 B/s, 0 objects/s recovering 2024-04-15T21:35:39.861 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -e sha1=a9a752df26c63acad72e1b3569fd79a515ca0765 -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-15T21:35:39.883 INFO:teuthology.orchestra.run.smithi083.stdout: 5 1916 18.87 MB/sec warmup 7 sec latency 23.616 ms 2024-04-15T21:35:40.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:39 smithi139 ceph-mon[26919]: pgmap v242: 97 pgs: 97 active+clean; 81 MiB data, 124 MiB used, 715 GiB / 715 GiB avail; 220 B/s rd, 8.9 MiB/s wr, 385 op/s; 0 B/s, 0 objects/s recovering 2024-04-15T21:35:40.884 INFO:teuthology.orchestra.run.smithi083.stdout: 5 2940 20.70 MB/sec warmup 8 sec latency 17.158 ms 2024-04-15T21:35:41.641 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:41 smithi083 ceph-mon[24790]: pgmap v243: 97 pgs: 97 active+clean; 90 MiB data, 133 MiB used, 715 GiB / 715 GiB avail; 198 B/s rd, 9.1 MiB/s wr, 371 op/s; 0 B/s, 0 objects/s recovering 2024-04-15T21:35:41.884 INFO:teuthology.orchestra.run.smithi083.stdout: 5 3682 21.57 MB/sec warmup 9 sec latency 19.770 ms 2024-04-15T21:35:42.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:41 smithi139 ceph-mon[26919]: pgmap v243: 97 pgs: 97 active+clean; 90 MiB data, 133 MiB used, 715 GiB / 715 GiB avail; 198 B/s rd, 9.1 MiB/s wr, 371 op/s; 0 B/s, 0 objects/s recovering 2024-04-15T21:35:42.884 INFO:teuthology.orchestra.run.smithi083.stdout: 5 3958 20.03 MB/sec warmup 10 sec latency 53.100 ms 2024-04-15T21:35:43.660 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:43 smithi083 ceph-mon[24790]: pgmap v244: 97 pgs: 97 active+clean; 120 MiB data, 167 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 11 MiB/s wr, 532 op/s; 0 B/s, 0 objects/s recovering 2024-04-15T21:35:43.884 INFO:teuthology.orchestra.run.smithi083.stdout: 5 4658 19.08 MB/sec warmup 11 sec latency 40.972 ms 2024-04-15T21:35:44.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:43 smithi139 ceph-mon[26919]: pgmap v244: 97 pgs: 97 active+clean; 120 MiB data, 167 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 11 MiB/s wr, 532 op/s; 0 B/s, 0 objects/s recovering 2024-04-15T21:35:44.886 INFO:teuthology.orchestra.run.smithi083.stdout: 5 5560 19.01 MB/sec warmup 12 sec latency 19.025 ms 2024-04-15T21:35:45.595 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -e sha1=a9a752df26c63acad72e1b3569fd79a515ca0765 -- bash -c 'ceph mgr module enable nfs --force' 2024-04-15T21:35:45.675 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:45 smithi083 ceph-mon[24790]: pgmap v245: 97 pgs: 97 active+clean; 128 MiB data, 219 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 13 MiB/s wr, 662 op/s; 0 B/s, 0 objects/s recovering 2024-04-15T21:35:45.884 INFO:teuthology.orchestra.run.smithi083.stdout: 5 6074 18.38 MB/sec warmup 13 sec latency 46.442 ms 2024-04-15T21:35:46.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:45 smithi139 ceph-mon[26919]: pgmap v245: 97 pgs: 97 active+clean; 128 MiB data, 219 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 13 MiB/s wr, 662 op/s; 0 B/s, 0 objects/s recovering 2024-04-15T21:35:46.884 INFO:teuthology.orchestra.run.smithi083.stdout: 5 6886 19.47 MB/sec warmup 14 sec latency 29.360 ms 2024-04-15T21:35:47.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:47 smithi083 ceph-mon[24790]: pgmap v246: 97 pgs: 97 active+clean; 128 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 14 MiB/s wr, 753 op/s 2024-04-15T21:35:47.884 INFO:teuthology.orchestra.run.smithi083.stdout: 5 7392 19.63 MB/sec warmup 15 sec latency 30.030 ms 2024-04-15T21:35:48.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:47 smithi139 ceph-mon[26919]: pgmap v246: 97 pgs: 97 active+clean; 128 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 14 MiB/s wr, 753 op/s 2024-04-15T21:35:48.885 INFO:teuthology.orchestra.run.smithi083.stdout: 5 7845 18.61 MB/sec warmup 16 sec latency 55.085 ms 2024-04-15T21:35:49.649 INFO:teuthology.orchestra.run.smithi083.stderr:module 'nfs' is already enabled 2024-04-15T21:35:49.723 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:49 smithi083 ceph-mon[24790]: pgmap v247: 97 pgs: 97 active+clean; 138 MiB data, 316 MiB used, 715 GiB / 715 GiB avail; 3.3 KiB/s rd, 14 MiB/s wr, 986 op/s 2024-04-15T21:35:49.723 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:49 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1576553496' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-15T21:35:49.884 INFO:teuthology.orchestra.run.smithi083.stdout: 5 8419 18.38 MB/sec warmup 17 sec latency 68.977 ms 2024-04-15T21:35:49.914 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:49 smithi139 ceph-mon[26919]: pgmap v247: 97 pgs: 97 active+clean; 138 MiB data, 316 MiB used, 715 GiB / 715 GiB avail; 3.3 KiB/s rd, 14 MiB/s wr, 986 op/s 2024-04-15T21:35:49.914 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:49 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/1576553496' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-15T21:35:50.734 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:50 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1576553496' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-04-15T21:35:50.734 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:50 smithi083 ceph-mon[24790]: mgrmap e21: smithi083.kxqarn(active, since 6m), standbys: smithi139.bqqykn 2024-04-15T21:35:50.884 INFO:teuthology.orchestra.run.smithi083.stdout: 5 9261 18.11 MB/sec warmup 18 sec latency 42.893 ms 2024-04-15T21:35:50.912 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -e sha1=a9a752df26c63acad72e1b3569fd79a515ca0765 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-15T21:35:51.096 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:50 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/1576553496' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-04-15T21:35:51.097 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:50 smithi139 ceph-mon[26919]: mgrmap e21: smithi083.kxqarn(active, since 6m), standbys: smithi139.bqqykn 2024-04-15T21:35:51.812 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:51 smithi083 ceph-mon[24790]: pgmap v248: 97 pgs: 97 active+clean; 138 MiB data, 326 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 9.7 MiB/s wr, 743 op/s 2024-04-15T21:35:51.885 INFO:teuthology.orchestra.run.smithi083.stdout: 5 9716 17.96 MB/sec warmup 19 sec latency 31.780 ms 2024-04-15T21:35:52.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:51 smithi139 ceph-mon[26919]: pgmap v248: 97 pgs: 97 active+clean; 138 MiB data, 326 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 9.7 MiB/s wr, 743 op/s 2024-04-15T21:35:52.885 INFO:teuthology.orchestra.run.smithi083.stdout: 5 10509 18.71 MB/sec warmup 20 sec latency 20.353 ms 2024-04-15T21:35:53.830 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:53 smithi083 ceph-mon[24790]: pgmap v249: 97 pgs: 97 active+clean; 145 MiB data, 378 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 11 MiB/s wr, 996 op/s 2024-04-15T21:35:53.884 INFO:teuthology.orchestra.run.smithi083.stdout: 5 11022 18.62 MB/sec warmup 21 sec latency 22.665 ms 2024-04-15T21:35:54.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:53 smithi139 ceph-mon[26919]: pgmap v249: 97 pgs: 97 active+clean; 145 MiB data, 378 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 11 MiB/s wr, 996 op/s 2024-04-15T21:35:54.160 INFO:teuthology.orchestra.run.smithi083.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a9a752df26c63acad72e1b3569fd79a515ca0765 2024-04-15T21:35:54.885 INFO:teuthology.orchestra.run.smithi083.stdout: 5 11573 17.97 MB/sec warmup 22 sec latency 21.423 ms 2024-04-15T21:35:55.180 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:55 smithi083 ceph-mon[24790]: from='client.14614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a9a752df26c63acad72e1b3569fd79a515ca0765", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:55.181 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:55 smithi083 ceph-mon[24790]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a9a752df26c63acad72e1b3569fd79a515ca0765 2024-04-15T21:35:55.181 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:55 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:55.181 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:55 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:35:55.181 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:55 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:55.181 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:55 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:35:55.181 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:55 smithi083 ceph-mon[24790]: pgmap v250: 97 pgs: 97 active+clean; 144 MiB data, 395 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 8.9 MiB/s wr, 849 op/s 2024-04-15T21:35:55.181 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:55 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:55.181 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:55 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:55.181 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:55 smithi083 ceph-mon[24790]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a9a752df26c63acad72e1b3569fd79a515ca0765 2024-04-15T21:35:55.563 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-15T21:35:55.564 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-04-15T21:35:55.564 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -e sha1=a9a752df26c63acad72e1b3569fd79a515ca0765 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-04-15T21:35:55.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:55 smithi139 ceph-mon[26919]: from='client.14614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a9a752df26c63acad72e1b3569fd79a515ca0765", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:35:55.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:55 smithi139 ceph-mon[26919]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a9a752df26c63acad72e1b3569fd79a515ca0765 2024-04-15T21:35:55.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:55 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:55.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:55 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:35:55.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:55 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:55.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:55 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:35:55.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:55 smithi139 ceph-mon[26919]: pgmap v250: 97 pgs: 97 active+clean; 144 MiB data, 395 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 8.9 MiB/s wr, 849 op/s 2024-04-15T21:35:55.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:55 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:55.613 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:55 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:55.614 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:55 smithi139 ceph-mon[26919]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a9a752df26c63acad72e1b3569fd79a515ca0765 2024-04-15T21:35:55.885 INFO:teuthology.orchestra.run.smithi083.stdout: 5 12181 18.14 MB/sec warmup 23 sec latency 30.814 ms 2024-04-15T21:35:56.885 INFO:teuthology.orchestra.run.smithi083.stdout: 5 12893 17.61 MB/sec warmup 24 sec latency 61.071 ms 2024-04-15T21:35:57.111 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:56 smithi083 ceph-mon[24790]: Upgrade: Paused due to UPGRADE_FAILED_PULL: Upgrade: failed to pull target image 2024-04-15T21:35:57.111 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:56 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:57.111 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:56 smithi083 ceph-mon[24790]: pgmap v251: 97 pgs: 97 active+clean; 144 MiB data, 396 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 7.9 MiB/s wr, 763 op/s 2024-04-15T21:35:57.111 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:56 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:35:57.111 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:56 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:57.112 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:56 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:35:57.112 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:56 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:57.112 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:56 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:57.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:56 smithi139 ceph-mon[26919]: Upgrade: Paused due to UPGRADE_FAILED_PULL: Upgrade: failed to pull target image 2024-04-15T21:35:57.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:56 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:57.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:56 smithi139 ceph-mon[26919]: pgmap v251: 97 pgs: 97 active+clean; 144 MiB data, 396 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 7.9 MiB/s wr, 763 op/s 2024-04-15T21:35:57.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:56 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:35:57.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:56 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:35:57.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:56 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:35:57.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:56 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:57.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:56 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:57.885 INFO:teuthology.orchestra.run.smithi083.stdout: 5 13264 17.48 MB/sec warmup 25 sec latency 36.767 ms 2024-04-15T21:35:58.130 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:57 smithi083 ceph-mon[24790]: Health check failed: Upgrade: failed to pull target image (UPGRADE_FAILED_PULL) 2024-04-15T21:35:58.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:57 smithi139 ceph-mon[26919]: Health check failed: Upgrade: failed to pull target image (UPGRADE_FAILED_PULL) 2024-04-15T21:35:58.815 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-04-15T21:35:58.885 INFO:teuthology.orchestra.run.smithi083.stdout: 5 13794 17.61 MB/sec warmup 26 sec latency 55.711 ms 2024-04-15T21:35:59.283 INFO:teuthology.orchestra.run.smithi083.stdout:"Error: UPGRADE_FAILED_PULL: Upgrade: failed to pull target image" 2024-04-15T21:35:59.656 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:59 smithi083 ceph-mon[24790]: pgmap v252: 97 pgs: 97 active+clean; 156 MiB data, 456 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 9.8 MiB/s wr, 1.00k op/s 2024-04-15T21:35:59.656 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:35:59 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:59.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:59 smithi139 ceph-mon[26919]: pgmap v252: 97 pgs: 97 active+clean; 156 MiB data, 456 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 9.8 MiB/s wr, 1.00k op/s 2024-04-15T21:35:59.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:35:59 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:35:59.885 INFO:teuthology.orchestra.run.smithi083.stdout: 5 14419 17.95 MB/sec warmup 27 sec latency 23.679 ms 2024-04-15T21:36:00.633 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -e sha1=a9a752df26c63acad72e1b3569fd79a515ca0765 -- bash -c 'ceph orch ps' 2024-04-15T21:36:00.639 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:00 smithi083 ceph-mon[24790]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:36:00.639 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:00 smithi083 ceph-mon[24790]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:36:00.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:00 smithi139 ceph-mon[26919]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:36:00.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:00 smithi139 ceph-mon[26919]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:36:00.885 INFO:teuthology.orchestra.run.smithi083.stdout: 5 14882 17.43 MB/sec warmup 28 sec latency 22.608 ms 2024-04-15T21:36:01.672 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:01 smithi083 ceph-mon[24790]: pgmap v253: 97 pgs: 97 active+clean; 155 MiB data, 466 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 7.7 MiB/s wr, 740 op/s 2024-04-15T21:36:01.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:01 smithi139 ceph-mon[26919]: pgmap v253: 97 pgs: 97 active+clean; 155 MiB data, 466 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 7.7 MiB/s wr, 740 op/s 2024-04-15T21:36:01.885 INFO:teuthology.orchestra.run.smithi083.stdout: 5 15692 17.59 MB/sec warmup 29 sec latency 18.925 ms 2024-04-15T21:36:02.885 INFO:teuthology.orchestra.run.smithi083.stdout: 5 16477 17.32 MB/sec warmup 30 sec latency 28.807 ms 2024-04-15T21:36:03.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:03 smithi083 ceph-mon[24790]: pgmap v254: 97 pgs: 97 active+clean; 163 MiB data, 521 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.8 MiB/s wr, 945 op/s 2024-04-15T21:36:03.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:03 smithi139 ceph-mon[26919]: pgmap v254: 97 pgs: 97 active+clean; 163 MiB data, 521 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.8 MiB/s wr, 945 op/s 2024-04-15T21:36:03.885 INFO:teuthology.orchestra.run.smithi083.stdout: 5 16918 17.29 MB/sec warmup 31 sec latency 23.260 ms 2024-04-15T21:36:03.937 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-15T21:36:03.937 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (5m) 33s ago 7m 24.5M - ba2b418f427c b36057f4bce8 2024-04-15T21:36:03.937 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (6m) 33s ago 6m 6405k - 17.2.0 e1d6a67b021e 016deb461f37 2024-04-15T21:36:03.937 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi139 smithi139 running (6m) 35s ago 6m 6405k - 17.2.0 e1d6a67b021e 3495d2d44d56 2024-04-15T21:36:03.937 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (5m) 33s ago 6m 51.8M - 8.3.5 dad864ee21e9 9218fa47bfde 2024-04-15T21:36:03.937 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.lgyynu smithi083 running (66s) 33s ago 65s 12.4M - 17.2.0 e1d6a67b021e 49f7e4719479 2024-04-15T21:36:03.937 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi139.akqukn smithi139 running (63s) 35s ago 63s 13.5M - 17.2.0 e1d6a67b021e 2018439f0754 2024-04-15T21:36:03.937 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.kxqarn smithi083 *:9283 running (8m) 33s ago 8m 459M - 17.2.0 e1d6a67b021e 9332afb3cbc0 2024-04-15T21:36:03.937 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi139.bqqykn smithi139 *:8443,9283 running (6m) 35s ago 6m 414M - 17.2.0 e1d6a67b021e af4b07488571 2024-04-15T21:36:03.937 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (8m) 33s ago 8m 47.1M 2048M 17.2.0 e1d6a67b021e 4e1e2c99fe19 2024-04-15T21:36:03.938 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi139 smithi139 running (6m) 35s ago 6m 37.1M 2048M 17.2.0 e1d6a67b021e 3e6950ee7383 2024-04-15T21:36:03.938 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.vuiewy smithi083 *:2049 running (44s) 33s ago 43s 58.0M - 4.0 e1d6a67b021e 0650c287a5fe 2024-04-15T21:36:03.938 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi139.nuvjad smithi139 *:2049 running (38s) 35s ago 38s 50.9M - 4.0 e1d6a67b021e fb2b215462a0 2024-04-15T21:36:03.938 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (6m) 33s ago 6m 15.1M - 1dbe0e931976 17bc16074f49 2024-04-15T21:36:03.938 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi139 smithi139 *:9100 running (6m) 35s ago 6m 17.2M - 1dbe0e931976 df8a6fcc6bbd 2024-04-15T21:36:03.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5m) 33s ago 5m 45.9M 3216M 17.2.0 e1d6a67b021e a5cc4573e142 2024-04-15T21:36:03.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5m) 33s ago 5m 42.5M 3216M 17.2.0 e1d6a67b021e 4a973ccf3730 2024-04-15T21:36:03.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4m) 33s ago 4m 41.0M 3216M 17.2.0 e1d6a67b021e e1038d2e7462 2024-04-15T21:36:03.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4m) 33s ago 4m 42.7M 3216M 17.2.0 e1d6a67b021e aad5a54fb42f 2024-04-15T21:36:03.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi139 running (3m) 35s ago 3m 41.6M 3984M 17.2.0 e1d6a67b021e ca06e69e5c77 2024-04-15T21:36:03.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi139 running (3m) 35s ago 3m 39.4M 3984M 17.2.0 e1d6a67b021e 9696759277a4 2024-04-15T21:36:03.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi139 running (2m) 35s ago 2m 41.6M 3984M 17.2.0 e1d6a67b021e 8108f3669964 2024-04-15T21:36:03.938 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi139 running (2m) 35s ago 2m 40.1M 3984M 17.2.0 e1d6a67b021e ae9b789871b1 2024-04-15T21:36:03.939 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (5m) 33s ago 5m 62.5M - 514e6a882f6e 10a02d16091d 2024-04-15T21:36:04.885 INFO:teuthology.orchestra.run.smithi083.stdout: 5 17622 17.65 MB/sec warmup 32 sec latency 23.773 ms 2024-04-15T21:36:05.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:05 smithi083 ceph-mon[24790]: pgmap v255: 97 pgs: 97 active+clean; 173 MiB data, 545 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.6 MiB/s wr, 797 op/s 2024-04-15T21:36:05.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:05 smithi083 ceph-mon[24790]: from='client.24359 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:36:05.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:05 smithi139 ceph-mon[26919]: pgmap v255: 97 pgs: 97 active+clean; 173 MiB data, 545 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.6 MiB/s wr, 797 op/s 2024-04-15T21:36:05.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:05 smithi139 ceph-mon[26919]: from='client.24359 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:36:05.885 INFO:teuthology.orchestra.run.smithi083.stdout: 5 18134 17.64 MB/sec warmup 33 sec latency 55.638 ms 2024-04-15T21:36:06.265 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -e sha1=a9a752df26c63acad72e1b3569fd79a515ca0765 -- bash -c 'ceph versions' 2024-04-15T21:36:06.885 INFO:teuthology.orchestra.run.smithi083.stdout: 5 18618 17.25 MB/sec warmup 34 sec latency 41.442 ms 2024-04-15T21:36:07.748 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:07 smithi083 ceph-mon[24790]: pgmap v256: 97 pgs: 97 active+clean; 173 MiB data, 577 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 7.6 MiB/s wr, 745 op/s 2024-04-15T21:36:07.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:07 smithi139 ceph-mon[26919]: pgmap v256: 97 pgs: 97 active+clean; 173 MiB data, 577 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 7.6 MiB/s wr, 745 op/s 2024-04-15T21:36:07.885 INFO:teuthology.orchestra.run.smithi083.stdout: 5 19161 17.13 MB/sec warmup 35 sec latency 33.191 ms 2024-04-15T21:36:08.886 INFO:teuthology.orchestra.run.smithi083.stdout: 5 20110 17.11 MB/sec warmup 36 sec latency 29.855 ms 2024-04-15T21:36:09.782 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:09 smithi083 ceph-mon[24790]: pgmap v257: 97 pgs: 97 active+clean; 182 MiB data, 648 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-04-15T21:36:09.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:09 smithi139 ceph-mon[26919]: pgmap v257: 97 pgs: 97 active+clean; 182 MiB data, 648 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-04-15T21:36:09.881 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-04-15T21:36:09.881 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-04-15T21:36:09.881 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-15T21:36:09.881 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-15T21:36:09.881 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-04-15T21:36:09.881 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-15T21:36:09.881 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-15T21:36:09.881 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-04-15T21:36:09.881 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-15T21:36:09.881 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-15T21:36:09.881 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-04-15T21:36:09.881 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-15T21:36:09.881 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-15T21:36:09.882 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-04-15T21:36:09.882 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-15T21:36:09.882 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-04-15T21:36:09.882 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-04-15T21:36:09.889 INFO:teuthology.orchestra.run.smithi083.stdout: 5 20843 17.50 MB/sec warmup 37 sec latency 36.804 ms 2024-04-15T21:36:10.678 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:10 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1691902728' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-15T21:36:10.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:10 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/1691902728' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-15T21:36:10.889 INFO:teuthology.orchestra.run.smithi083.stdout: 5 21571 17.78 MB/sec warmup 38 sec latency 22.262 ms 2024-04-15T21:36:11.358 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -e sha1=a9a752df26c63acad72e1b3569fd79a515ca0765 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-04-15T21:36:11.681 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:11 smithi083 ceph-mon[24790]: pgmap v258: 97 pgs: 97 active+clean; 183 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 8.1 MiB/s wr, 731 op/s 2024-04-15T21:36:11.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:11 smithi139 ceph-mon[26919]: pgmap v258: 97 pgs: 97 active+clean; 183 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 8.1 MiB/s wr, 731 op/s 2024-04-15T21:36:11.889 INFO:teuthology.orchestra.run.smithi083.stdout: 5 22093 17.45 MB/sec warmup 39 sec latency 22.585 ms 2024-04-15T21:36:12.889 INFO:teuthology.orchestra.run.smithi083.stdout: 5 22599 17.37 MB/sec warmup 40 sec latency 49.573 ms 2024-04-15T21:36:13.708 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:13 smithi083 ceph-mon[24790]: pgmap v259: 97 pgs: 97 active+clean; 182 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 792 op/s 2024-04-15T21:36:13.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:13 smithi139 ceph-mon[26919]: pgmap v259: 97 pgs: 97 active+clean; 182 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 792 op/s 2024-04-15T21:36:13.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 23342 17.26 MB/sec warmup 41 sec latency 20.272 ms 2024-04-15T21:36:14.520 INFO:teuthology.orchestra.run.smithi083.stdout:wait for servicemap items w/ changing names to refresh 2024-04-15T21:36:14.889 INFO:teuthology.orchestra.run.smithi083.stdout: 5 23888 17.14 MB/sec warmup 42 sec latency 26.855 ms 2024-04-15T21:36:15.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:15 smithi083 ceph-mon[24790]: pgmap v260: 97 pgs: 97 active+clean; 187 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.5 MiB/s wr, 755 op/s 2024-04-15T21:36:15.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:15 smithi139 ceph-mon[26919]: pgmap v260: 97 pgs: 97 active+clean; 187 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.5 MiB/s wr, 755 op/s 2024-04-15T21:36:15.889 INFO:teuthology.orchestra.run.smithi083.stdout: 5 24513 17.40 MB/sec warmup 43 sec latency 57.732 ms 2024-04-15T21:36:16.292 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -e sha1=a9a752df26c63acad72e1b3569fd79a515ca0765 -- bash -c 'sleep 60' 2024-04-15T21:36:16.889 INFO:teuthology.orchestra.run.smithi083.stdout: 5 25092 17.51 MB/sec warmup 44 sec latency 22.013 ms 2024-04-15T21:36:17.739 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:17 smithi083 ceph-mon[24790]: pgmap v261: 97 pgs: 97 active+clean; 191 MiB data, 721 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 8.0 MiB/s wr, 711 op/s 2024-04-15T21:36:17.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:17 smithi139 ceph-mon[26919]: pgmap v261: 97 pgs: 97 active+clean; 191 MiB data, 721 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 8.0 MiB/s wr, 711 op/s 2024-04-15T21:36:17.889 INFO:teuthology.orchestra.run.smithi083.stdout: 5 25442 17.22 MB/sec warmup 45 sec latency 42.101 ms 2024-04-15T21:36:18.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 25948 17.05 MB/sec warmup 46 sec latency 61.610 ms 2024-04-15T21:36:19.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:19 smithi083 ceph-mon[24790]: pgmap v262: 97 pgs: 97 active+clean; 197 MiB data, 793 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 990 op/s 2024-04-15T21:36:19.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:19 smithi139 ceph-mon[26919]: pgmap v262: 97 pgs: 97 active+clean; 197 MiB data, 793 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 990 op/s 2024-04-15T21:36:19.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 26690 17.04 MB/sec warmup 47 sec latency 38.256 ms 2024-04-15T21:36:20.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 27444 17.01 MB/sec warmup 48 sec latency 29.198 ms 2024-04-15T21:36:21.794 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:21 smithi083 ceph-mon[24790]: pgmap v263: 97 pgs: 97 active+clean; 204 MiB data, 794 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.2 MiB/s wr, 744 op/s 2024-04-15T21:36:21.862 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:21 smithi139 ceph-mon[26919]: pgmap v263: 97 pgs: 97 active+clean; 204 MiB data, 794 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.2 MiB/s wr, 744 op/s 2024-04-15T21:36:21.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 28362 17.42 MB/sec warmup 49 sec latency 22.703 ms 2024-04-15T21:36:22.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 28969 17.41 MB/sec warmup 50 sec latency 21.616 ms 2024-04-15T21:36:23.849 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:23 smithi083 ceph-mon[24790]: pgmap v264: 97 pgs: 97 active+clean; 208 MiB data, 825 MiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 9.8 MiB/s wr, 884 op/s 2024-04-15T21:36:23.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:23 smithi139 ceph-mon[26919]: pgmap v264: 97 pgs: 97 active+clean; 208 MiB data, 825 MiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 9.8 MiB/s wr, 884 op/s 2024-04-15T21:36:23.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 29479 17.21 MB/sec warmup 51 sec latency 43.845 ms 2024-04-15T21:36:24.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 30164 17.24 MB/sec warmup 52 sec latency 33.074 ms 2024-04-15T21:36:25.624 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:25 smithi083 ceph-mon[24790]: pgmap v265: 97 pgs: 97 active+clean; 212 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 9.2 MiB/s wr, 952 op/s 2024-04-15T21:36:25.863 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:25 smithi139 ceph-mon[26919]: pgmap v265: 97 pgs: 97 active+clean; 212 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 9.2 MiB/s wr, 952 op/s 2024-04-15T21:36:25.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 30826 17.09 MB/sec warmup 53 sec latency 30.189 ms 2024-04-15T21:36:26.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 31577 17.39 MB/sec warmup 54 sec latency 45.364 ms 2024-04-15T21:36:27.645 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:27 smithi083 ceph-mon[24790]: pgmap v266: 97 pgs: 97 active+clean; 211 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.4 MiB/s wr, 799 op/s 2024-04-15T21:36:27.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 32207 17.51 MB/sec warmup 55 sec latency 23.454 ms 2024-04-15T21:36:28.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:27 smithi139 ceph-mon[26919]: pgmap v266: 97 pgs: 97 active+clean; 211 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.4 MiB/s wr, 799 op/s 2024-04-15T21:36:28.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:36:28.793 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:36:28.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 32649 17.26 MB/sec warmup 56 sec latency 35.132 ms 2024-04-15T21:36:29.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:36:29.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:36:29.784 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:29 smithi083 ceph-mon[24790]: pgmap v267: 97 pgs: 97 active+clean; 223 MiB data, 954 MiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 11 MiB/s wr, 1.09k op/s 2024-04-15T21:36:29.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 33152 17.17 MB/sec warmup 57 sec latency 44.198 ms 2024-04-15T21:36:30.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:29 smithi139 ceph-mon[26919]: pgmap v267: 97 pgs: 97 active+clean; 223 MiB data, 954 MiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 11 MiB/s wr, 1.09k op/s 2024-04-15T21:36:30.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 33949 17.14 MB/sec warmup 58 sec latency 93.710 ms 2024-04-15T21:36:31.841 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:31 smithi083 ceph-mon[24790]: pgmap v268: 97 pgs: 97 active+clean; 221 MiB data, 954 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.5 MiB/s wr, 808 op/s 2024-04-15T21:36:31.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 34584 17.11 MB/sec warmup 59 sec latency 121.573 ms 2024-04-15T21:36:32.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:31 smithi139 ceph-mon[26919]: pgmap v268: 97 pgs: 97 active+clean; 221 MiB data, 954 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.5 MiB/s wr, 808 op/s 2024-04-15T21:36:32.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 35482 17.47 MB/sec warmup 60 sec latency 19.552 ms 2024-04-15T21:36:33.861 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:33 smithi083 ceph-mon[24790]: pgmap v269: 97 pgs: 97 active+clean; 223 MiB data, 967 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.5 MiB/s wr, 895 op/s 2024-04-15T21:36:33.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 35949 17.40 MB/sec warmup 61 sec latency 32.020 ms 2024-04-15T21:36:34.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:33 smithi139 ceph-mon[26919]: pgmap v269: 97 pgs: 97 active+clean; 223 MiB data, 967 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.5 MiB/s wr, 895 op/s 2024-04-15T21:36:34.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 36406 17.18 MB/sec warmup 62 sec latency 34.539 ms 2024-04-15T21:36:35.656 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:35 smithi083 ceph-mon[24790]: pgmap v270: 97 pgs: 97 active+clean; 227 MiB data, 995 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 912 op/s 2024-04-15T21:36:35.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 37190 17.26 MB/sec warmup 63 sec latency 22.279 ms 2024-04-15T21:36:36.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:35 smithi139 ceph-mon[26919]: pgmap v270: 97 pgs: 97 active+clean; 227 MiB data, 995 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 912 op/s 2024-04-15T21:36:36.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 38020 17.21 MB/sec warmup 64 sec latency 18.724 ms 2024-04-15T21:36:37.664 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:37 smithi083 ceph-mon[24790]: pgmap v271: 97 pgs: 97 active+clean; 227 MiB data, 1018 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.1 MiB/s wr, 788 op/s 2024-04-15T21:36:37.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 38922 17.52 MB/sec warmup 65 sec latency 22.287 ms 2024-04-15T21:36:38.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:37 smithi139 ceph-mon[26919]: pgmap v271: 97 pgs: 97 active+clean; 227 MiB data, 1018 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.1 MiB/s wr, 788 op/s 2024-04-15T21:36:38.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 39466 17.52 MB/sec warmup 66 sec latency 23.249 ms 2024-04-15T21:36:39.675 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:39 smithi083 ceph-mon[24790]: pgmap v272: 97 pgs: 97 active+clean; 242 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 980 op/s 2024-04-15T21:36:39.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 40036 17.35 MB/sec warmup 67 sec latency 65.284 ms 2024-04-15T21:36:40.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:39 smithi139 ceph-mon[26919]: pgmap v272: 97 pgs: 97 active+clean; 242 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 980 op/s 2024-04-15T21:36:40.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 40799 17.40 MB/sec warmup 68 sec latency 21.127 ms 2024-04-15T21:36:41.709 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:41 smithi083 ceph-mon[24790]: pgmap v273: 97 pgs: 97 active+clean; 252 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.6 MiB/s wr, 777 op/s 2024-04-15T21:36:41.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 41481 17.28 MB/sec warmup 69 sec latency 32.845 ms 2024-04-15T21:36:42.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:41 smithi139 ceph-mon[26919]: pgmap v273: 97 pgs: 97 active+clean; 252 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.6 MiB/s wr, 777 op/s 2024-04-15T21:36:42.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 42253 17.49 MB/sec warmup 70 sec latency 42.858 ms 2024-04-15T21:36:43.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:43 smithi083 ceph-mon[24790]: pgmap v274: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 976 op/s 2024-04-15T21:36:43.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 42880 17.62 MB/sec warmup 71 sec latency 43.887 ms 2024-04-15T21:36:44.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:43 smithi139 ceph-mon[26919]: pgmap v274: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 976 op/s 2024-04-15T21:36:44.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 43322 17.41 MB/sec warmup 72 sec latency 29.911 ms 2024-04-15T21:36:45.870 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:45 smithi083 ceph-mon[24790]: pgmap v275: 97 pgs: 97 active+clean; 252 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 875 op/s 2024-04-15T21:36:45.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 43689 17.27 MB/sec warmup 73 sec latency 38.122 ms 2024-04-15T21:36:46.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:45 smithi139 ceph-mon[26919]: pgmap v275: 97 pgs: 97 active+clean; 252 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 875 op/s 2024-04-15T21:36:46.890 INFO:teuthology.orchestra.run.smithi083.stdout: 5 44293 17.28 MB/sec warmup 74 sec latency 31.888 ms 2024-04-15T21:36:47.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 44993 17.16 MB/sec warmup 75 sec latency 30.351 ms 2024-04-15T21:36:47.896 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:47 smithi083 ceph-mon[24790]: pgmap v276: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 8.6 MiB/s wr, 735 op/s 2024-04-15T21:36:48.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:47 smithi139 ceph-mon[26919]: pgmap v276: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 8.6 MiB/s wr, 735 op/s 2024-04-15T21:36:48.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 45610 17.30 MB/sec warmup 76 sec latency 32.243 ms 2024-04-15T21:36:49.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 46334 17.45 MB/sec warmup 77 sec latency 33.188 ms 2024-04-15T21:36:49.914 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:49 smithi083 ceph-mon[24790]: pgmap v277: 97 pgs: 97 active+clean; 260 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-04-15T21:36:50.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:49 smithi139 ceph-mon[26919]: pgmap v277: 97 pgs: 97 active+clean; 260 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-04-15T21:36:50.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 46709 17.29 MB/sec warmup 78 sec latency 36.138 ms 2024-04-15T21:36:51.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:51 smithi083 ceph-mon[24790]: pgmap v278: 97 pgs: 97 active+clean; 260 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.2 MiB/s wr, 814 op/s 2024-04-15T21:36:51.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 47244 17.17 MB/sec warmup 79 sec latency 24.647 ms 2024-04-15T21:36:52.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:51 smithi139 ceph-mon[26919]: pgmap v278: 97 pgs: 97 active+clean; 260 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.2 MiB/s wr, 814 op/s 2024-04-15T21:36:52.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 47978 17.18 MB/sec warmup 80 sec latency 59.608 ms 2024-04-15T21:36:53.703 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:53 smithi083 ceph-mon[24790]: pgmap v279: 97 pgs: 97 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 832 op/s 2024-04-15T21:36:53.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 48587 17.09 MB/sec warmup 81 sec latency 84.905 ms 2024-04-15T21:36:54.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:53 smithi139 ceph-mon[26919]: pgmap v279: 97 pgs: 97 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 832 op/s 2024-04-15T21:36:54.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 49294 17.24 MB/sec warmup 82 sec latency 28.495 ms 2024-04-15T21:36:55.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:55 smithi083 ceph-mon[24790]: pgmap v280: 97 pgs: 97 active+clean; 276 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 735 op/s 2024-04-15T21:36:55.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 49970 17.37 MB/sec warmup 83 sec latency 21.989 ms 2024-04-15T21:36:55.957 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:55 smithi139 ceph-mon[26919]: pgmap v280: 97 pgs: 97 active+clean; 276 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 735 op/s 2024-04-15T21:36:56.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:56 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:36:56.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:56 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:36:56.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:56 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:36:56.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:56 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:36:56.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:56 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:36:56.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 50512 17.21 MB/sec warmup 84 sec latency 26.979 ms 2024-04-15T21:36:57.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:56 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-15T21:36:57.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:56 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-15T21:36:57.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:56 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-15T21:36:57.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:56 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:36:57.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:56 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' 2024-04-15T21:36:57.735 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:57 smithi083 ceph-mon[24790]: pgmap v281: 97 pgs: 97 active+clean; 275 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.5 MiB/s wr, 679 op/s 2024-04-15T21:36:57.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 51237 17.28 MB/sec warmup 85 sec latency 24.885 ms 2024-04-15T21:36:58.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:57 smithi139 ceph-mon[26919]: pgmap v281: 97 pgs: 97 active+clean; 275 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.5 MiB/s wr, 679 op/s 2024-04-15T21:36:58.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 52118 17.20 MB/sec warmup 86 sec latency 18.106 ms 2024-04-15T21:36:59.751 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:36:59 smithi083 ceph-mon[24790]: pgmap v282: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 980 op/s 2024-04-15T21:36:59.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 52817 17.36 MB/sec warmup 87 sec latency 25.760 ms 2024-04-15T21:37:00.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:36:59 smithi139 ceph-mon[26919]: pgmap v282: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 980 op/s 2024-04-15T21:37:00.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 53460 17.45 MB/sec warmup 88 sec latency 64.515 ms 2024-04-15T21:37:01.800 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:01 smithi083 ceph-mon[24790]: pgmap v283: 97 pgs: 97 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.3 MiB/s wr, 781 op/s 2024-04-15T21:37:01.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 53918 17.31 MB/sec warmup 89 sec latency 31.595 ms 2024-04-15T21:37:02.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:01 smithi139 ceph-mon[26919]: pgmap v283: 97 pgs: 97 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.3 MiB/s wr, 781 op/s 2024-04-15T21:37:02.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 54413 17.23 MB/sec warmup 90 sec latency 47.104 ms 2024-04-15T21:37:03.735 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:03 smithi083 ceph-mon[24790]: pgmap v284: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-04-15T21:37:03.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 55141 17.22 MB/sec warmup 91 sec latency 37.019 ms 2024-04-15T21:37:04.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:03 smithi139 ceph-mon[26919]: pgmap v284: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-04-15T21:37:04.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 55821 17.21 MB/sec warmup 92 sec latency 28.794 ms 2024-04-15T21:37:05.754 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:05 smithi083 ceph-mon[24790]: pgmap v285: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 912 op/s 2024-04-15T21:37:05.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 56467 17.31 MB/sec warmup 93 sec latency 31.263 ms 2024-04-15T21:37:06.112 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:05 smithi139 ceph-mon[26919]: pgmap v285: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 912 op/s 2024-04-15T21:37:06.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 57035 17.36 MB/sec warmup 94 sec latency 31.437 ms 2024-04-15T21:37:07.771 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:07 smithi083 ceph-mon[24790]: pgmap v286: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.7 MiB/s wr, 809 op/s 2024-04-15T21:37:07.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 57430 17.22 MB/sec warmup 95 sec latency 47.940 ms 2024-04-15T21:37:08.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:07 smithi139 ceph-mon[26919]: pgmap v286: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.7 MiB/s wr, 809 op/s 2024-04-15T21:37:08.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 57874 17.13 MB/sec warmup 96 sec latency 33.926 ms 2024-04-15T21:37:09.786 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:09 smithi083 ceph-mon[24790]: pgmap v287: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 1.13k op/s 2024-04-15T21:37:09.891 INFO:teuthology.orchestra.run.smithi083.stdout: 5 58469 17.12 MB/sec warmup 97 sec latency 30.666 ms 2024-04-15T21:37:10.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:09 smithi139 ceph-mon[26919]: pgmap v287: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 1.13k op/s 2024-04-15T21:37:10.892 INFO:teuthology.orchestra.run.smithi083.stdout: 5 59212 17.06 MB/sec warmup 98 sec latency 26.734 ms 2024-04-15T21:37:11.806 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:11 smithi083 ceph-mon[24790]: pgmap v288: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.5 MiB/s wr, 803 op/s 2024-04-15T21:37:11.892 INFO:teuthology.orchestra.run.smithi083.stdout: 5 59893 17.19 MB/sec warmup 99 sec latency 37.221 ms 2024-04-15T21:37:12.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:11 smithi139 ceph-mon[26919]: pgmap v288: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.5 MiB/s wr, 803 op/s 2024-04-15T21:37:12.892 INFO:teuthology.orchestra.run.smithi083.stdout: 5 60523 17.26 MB/sec warmup 100 sec latency 29.807 ms 2024-04-15T21:37:13.835 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:13 smithi083 ceph-mon[24790]: pgmap v289: 97 pgs: 97 active+clean; 312 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 9.6 MiB/s wr, 916 op/s 2024-04-15T21:37:13.892 INFO:teuthology.orchestra.run.smithi083.stdout: 5 60868 17.14 MB/sec warmup 101 sec latency 61.907 ms 2024-04-15T21:37:14.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:13 smithi139 ceph-mon[26919]: pgmap v289: 97 pgs: 97 active+clean; 312 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 9.6 MiB/s wr, 916 op/s 2024-04-15T21:37:14.892 INFO:teuthology.orchestra.run.smithi083.stdout: 5 61319 17.02 MB/sec warmup 102 sec latency 36.740 ms 2024-04-15T21:37:15.886 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:15 smithi083 ceph-mon[24790]: pgmap v290: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.9 MiB/s wr, 746 op/s 2024-04-15T21:37:15.892 INFO:teuthology.orchestra.run.smithi083.stdout: 5 61816 16.99 MB/sec warmup 103 sec latency 38.587 ms 2024-04-15T21:37:16.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:15 smithi139 ceph-mon[26919]: pgmap v290: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.9 MiB/s wr, 746 op/s 2024-04-15T21:37:16.892 INFO:teuthology.orchestra.run.smithi083.stdout: 5 62473 16.94 MB/sec warmup 104 sec latency 34.136 ms 2024-04-15T21:37:17.892 INFO:teuthology.orchestra.run.smithi083.stdout: 5 63010 16.92 MB/sec warmup 105 sec latency 34.187 ms 2024-04-15T21:37:17.922 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:17 smithi083 ceph-mon[24790]: pgmap v291: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.2 MiB/s wr, 668 op/s 2024-04-15T21:37:18.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:17 smithi139 ceph-mon[26919]: pgmap v291: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.2 MiB/s wr, 668 op/s 2024-04-15T21:37:18.892 INFO:teuthology.orchestra.run.smithi083.stdout: 5 63577 17.00 MB/sec warmup 106 sec latency 74.163 ms 2024-04-15T21:37:19.892 INFO:teuthology.orchestra.run.smithi083.stdout: 5 64134 17.03 MB/sec warmup 107 sec latency 25.496 ms 2024-04-15T21:37:19.945 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:19 smithi083 ceph-mon[24790]: pgmap v292: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 9.7 MiB/s wr, 979 op/s 2024-04-15T21:37:20.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:19 smithi139 ceph-mon[26919]: pgmap v292: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 9.7 MiB/s wr, 979 op/s 2024-04-15T21:37:20.892 INFO:teuthology.orchestra.run.smithi083.stdout: 5 64558 16.93 MB/sec warmup 108 sec latency 36.976 ms 2024-04-15T21:37:21.740 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -e sha1=a9a752df26c63acad72e1b3569fd79a515ca0765 -- bash -c 'ceph orch ps' 2024-04-15T21:37:21.892 INFO:teuthology.orchestra.run.smithi083.stdout: 5 65056 16.88 MB/sec warmup 109 sec latency 26.772 ms 2024-04-15T21:37:21.977 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:21 smithi083 ceph-mon[24790]: pgmap v293: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 6.8 MiB/s wr, 675 op/s 2024-04-15T21:37:22.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:21 smithi139 ceph-mon[26919]: pgmap v293: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 6.8 MiB/s wr, 675 op/s 2024-04-15T21:37:22.892 INFO:teuthology.orchestra.run.smithi083.stdout: 5 65562 16.80 MB/sec warmup 110 sec latency 50.039 ms 2024-04-15T21:37:23.854 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:23 smithi083 ceph-mon[24790]: pgmap v294: 97 pgs: 97 active+clean; 324 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 8.0 MiB/s wr, 783 op/s 2024-04-15T21:37:23.892 INFO:teuthology.orchestra.run.smithi083.stdout: 5 66223 16.78 MB/sec warmup 111 sec latency 41.919 ms 2024-04-15T21:37:24.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:23 smithi139 ceph-mon[26919]: pgmap v294: 97 pgs: 97 active+clean; 324 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 8.0 MiB/s wr, 783 op/s 2024-04-15T21:37:24.893 INFO:teuthology.orchestra.run.smithi083.stdout: 5 66711 16.77 MB/sec warmup 112 sec latency 44.395 ms 2024-04-15T21:37:25.172 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-15T21:37:25.173 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (6m) 114s ago 8m 24.5M - ba2b418f427c b36057f4bce8 2024-04-15T21:37:25.173 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (8m) 114s ago 8m 6405k - 17.2.0 e1d6a67b021e 016deb461f37 2024-04-15T21:37:25.173 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi139 smithi139 running (7m) 116s ago 7m 6405k - 17.2.0 e1d6a67b021e 3495d2d44d56 2024-04-15T21:37:25.173 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (6m) 114s ago 8m 51.8M - 8.3.5 dad864ee21e9 9218fa47bfde 2024-04-15T21:37:25.173 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.lgyynu smithi083 running (2m) 114s ago 2m 12.4M - 17.2.0 e1d6a67b021e 49f7e4719479 2024-04-15T21:37:25.173 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi139.akqukn smithi139 running (2m) 116s ago 2m 13.5M - 17.2.0 e1d6a67b021e 2018439f0754 2024-04-15T21:37:25.173 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.kxqarn smithi083 *:9283 running (9m) 114s ago 9m 459M - 17.2.0 e1d6a67b021e 9332afb3cbc0 2024-04-15T21:37:25.173 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi139.bqqykn smithi139 *:8443,9283 running (7m) 116s ago 7m 414M - 17.2.0 e1d6a67b021e af4b07488571 2024-04-15T21:37:25.173 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (9m) 114s ago 9m 47.1M 2048M 17.2.0 e1d6a67b021e 4e1e2c99fe19 2024-04-15T21:37:25.173 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi139 smithi139 running (7m) 116s ago 7m 37.1M 2048M 17.2.0 e1d6a67b021e 3e6950ee7383 2024-04-15T21:37:25.173 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.vuiewy smithi083 *:2049 running (2m) 114s ago 2m 58.0M - 4.0 e1d6a67b021e 0650c287a5fe 2024-04-15T21:37:25.173 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi139.nuvjad smithi139 *:2049 running (2m) 116s ago 119s 50.9M - 4.0 e1d6a67b021e fb2b215462a0 2024-04-15T21:37:25.174 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (8m) 114s ago 8m 15.1M - 1dbe0e931976 17bc16074f49 2024-04-15T21:37:25.174 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi139 smithi139 *:9100 running (7m) 116s ago 7m 17.2M - 1dbe0e931976 df8a6fcc6bbd 2024-04-15T21:37:25.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6m) 114s ago 6m 45.9M 3216M 17.2.0 e1d6a67b021e a5cc4573e142 2024-04-15T21:37:25.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6m) 114s ago 6m 42.5M 3216M 17.2.0 e1d6a67b021e 4a973ccf3730 2024-04-15T21:37:25.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5m) 114s ago 5m 41.0M 3216M 17.2.0 e1d6a67b021e e1038d2e7462 2024-04-15T21:37:25.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5m) 114s ago 5m 42.7M 3216M 17.2.0 e1d6a67b021e aad5a54fb42f 2024-04-15T21:37:25.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi139 running (5m) 116s ago 5m 41.6M 3984M 17.2.0 e1d6a67b021e ca06e69e5c77 2024-04-15T21:37:25.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi139 running (4m) 116s ago 4m 39.4M 3984M 17.2.0 e1d6a67b021e 9696759277a4 2024-04-15T21:37:25.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi139 running (4m) 116s ago 4m 41.6M 3984M 17.2.0 e1d6a67b021e 8108f3669964 2024-04-15T21:37:25.174 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi139 running (3m) 116s ago 3m 40.1M 3984M 17.2.0 e1d6a67b021e ae9b789871b1 2024-04-15T21:37:25.174 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (7m) 114s ago 7m 62.5M - 514e6a882f6e 10a02d16091d 2024-04-15T21:37:25.876 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:25 smithi083 ceph-mon[24790]: pgmap v295: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.4 MiB/s wr, 740 op/s 2024-04-15T21:37:25.893 INFO:teuthology.orchestra.run.smithi083.stdout: 5 67237 16.83 MB/sec warmup 113 sec latency 32.991 ms 2024-04-15T21:37:26.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:25 smithi139 ceph-mon[26919]: pgmap v295: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.4 MiB/s wr, 740 op/s 2024-04-15T21:37:26.414 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -e sha1=a9a752df26c63acad72e1b3569fd79a515ca0765 -- bash -c 'ceph orch upgrade status' 2024-04-15T21:37:26.884 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:26 smithi083 ceph-mon[24790]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:37:26.896 INFO:teuthology.orchestra.run.smithi083.stdout: 5 67698 16.83 MB/sec warmup 114 sec latency 54.725 ms 2024-04-15T21:37:27.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:26 smithi139 ceph-mon[26919]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:37:27.897 INFO:teuthology.orchestra.run.smithi083.stdout: 5 68000 16.71 MB/sec warmup 115 sec latency 40.278 ms 2024-04-15T21:37:27.899 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:27 smithi083 ceph-mon[24790]: pgmap v296: 97 pgs: 97 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 6.9 MiB/s wr, 682 op/s 2024-04-15T21:37:28.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:27 smithi139 ceph-mon[26919]: pgmap v296: 97 pgs: 97 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 6.9 MiB/s wr, 682 op/s 2024-04-15T21:37:28.896 INFO:teuthology.orchestra.run.smithi083.stdout: 5 68417 16.64 MB/sec warmup 116 sec latency 50.779 ms 2024-04-15T21:37:28.933 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:37:28.934 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:28 smithi083 ceph-mon[24790]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:37:29.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/trash_purge_schedule"}]: dispatch 2024-04-15T21:37:29.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:28 smithi139 ceph-mon[26919]: from='mgr.14200 172.21.15.83:0/189683653' entity='mgr.smithi083.kxqarn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.kxqarn/mirror_snapshot_schedule"}]: dispatch 2024-04-15T21:37:29.390 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-04-15T21:37:29.390 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:a9a752df26c63acad72e1b3569fd79a515ca0765", 2024-04-15T21:37:29.390 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-04-15T21:37:29.390 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [], 2024-04-15T21:37:29.390 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "", 2024-04-15T21:37:29.390 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Error: UPGRADE_FAILED_PULL: Upgrade: failed to pull target image" 2024-04-15T21:37:29.390 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-04-15T21:37:29.896 INFO:teuthology.orchestra.run.smithi083.stdout: 5 68951 16.59 MB/sec warmup 117 sec latency 61.330 ms 2024-04-15T21:37:29.978 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:29 smithi083 ceph-mon[24790]: pgmap v297: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 9.8 MiB/s wr, 1.02k op/s 2024-04-15T21:37:30.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:29 smithi139 ceph-mon[26919]: pgmap v297: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 9.8 MiB/s wr, 1.02k op/s 2024-04-15T21:37:30.527 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -e sha1=a9a752df26c63acad72e1b3569fd79a515ca0765 -- bash -c 'ceph health detail' 2024-04-15T21:37:30.897 INFO:teuthology.orchestra.run.smithi083.stdout: 5 69649 16.58 MB/sec warmup 118 sec latency 27.582 ms 2024-04-15T21:37:30.989 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:30 smithi083 ceph-mon[24790]: from='client.14638 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:37:31.096 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:30 smithi139 ceph-mon[26919]: from='client.14638 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-15T21:37:31.897 INFO:teuthology.orchestra.run.smithi083.stdout: 5 70351 16.64 MB/sec warmup 119 sec latency 26.086 ms 2024-04-15T21:37:31.995 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:31 smithi083 ceph-mon[24790]: pgmap v298: 97 pgs: 97 active+clean; 342 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.2 MiB/s wr, 720 op/s 2024-04-15T21:37:32.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:31 smithi139 ceph-mon[26919]: pgmap v298: 97 pgs: 97 active+clean; 342 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.2 MiB/s wr, 720 op/s 2024-04-15T21:37:33.798 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:33 smithi083 ceph-mon[24790]: pgmap v299: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.9 MiB/s wr, 748 op/s 2024-04-15T21:37:33.897 INFO:teuthology.orchestra.run.smithi083.stdout: 5 71448 11.97 MB/sec execute 1 sec latency 45.092 ms 2024-04-15T21:37:34.082 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_WARN Upgrade: failed to pull target image 2024-04-15T21:37:34.082 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] UPGRADE_FAILED_PULL: Upgrade: failed to pull target image 2024-04-15T21:37:34.082 INFO:teuthology.orchestra.run.smithi083.stdout: host smithi083 `cephadm pull` failed: cephadm exited with an error code: 1, stderr: Pulling container image quay.ceph.io/ceph-ci/ceph:a9a752df26c63acad72e1b3569fd79a515ca0765... 2024-04-15T21:37:34.082 INFO:teuthology.orchestra.run.smithi083.stdout:Non-zero exit code 125 from /bin/podman pull quay.ceph.io/ceph-ci/ceph:a9a752df26c63acad72e1b3569fd79a515ca0765 2024-04-15T21:37:34.083 INFO:teuthology.orchestra.run.smithi083.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:a9a752df26c63acad72e1b3569fd79a515ca0765... 2024-04-15T21:37:34.083 INFO:teuthology.orchestra.run.smithi083.stdout:/bin/podman: stderr Error: initializing source docker://quay.ceph.io/ceph-ci/ceph:a9a752df26c63acad72e1b3569fd79a515ca0765: reading manifest a9a752df26c63acad72e1b3569fd79a515ca0765 in quay.ceph.io/ceph-ci/ceph: manifest unknown 2024-04-15T21:37:34.083 INFO:teuthology.orchestra.run.smithi083.stdout:ERROR: Failed command: /bin/podman pull quay.ceph.io/ceph-ci/ceph:a9a752df26c63acad72e1b3569fd79a515ca0765 2024-04-15T21:37:34.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:33 smithi139 ceph-mon[26919]: pgmap v299: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.9 MiB/s wr, 748 op/s 2024-04-15T21:37:34.804 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:34 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/3435565810' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-15T21:37:34.898 INFO:teuthology.orchestra.run.smithi083.stdout: 5 71839 8.28 MB/sec execute 2 sec latency 57.232 ms 2024-04-15T21:37:35.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:34 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/3435565810' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-15T21:37:35.193 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -e sha1=a9a752df26c63acad72e1b3569fd79a515ca0765 -- bash -c 'ceph versions' 2024-04-15T21:37:35.819 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:35 smithi083 ceph-mon[24790]: pgmap v300: 97 pgs: 97 active+clean; 328 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.3 MiB/s wr, 720 op/s 2024-04-15T21:37:35.897 INFO:teuthology.orchestra.run.smithi083.stdout: 5 72361 9.08 MB/sec execute 3 sec latency 34.422 ms 2024-04-15T21:37:36.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:35 smithi139 ceph-mon[26919]: pgmap v300: 97 pgs: 97 active+clean; 328 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.3 MiB/s wr, 720 op/s 2024-04-15T21:37:36.897 INFO:teuthology.orchestra.run.smithi083.stdout: 5 72962 10.48 MB/sec execute 4 sec latency 29.126 ms 2024-04-15T21:37:37.849 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:37 smithi083 ceph-mon[24790]: pgmap v301: 97 pgs: 97 active+clean; 333 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 6.8 MiB/s wr, 651 op/s 2024-04-15T21:37:37.898 INFO:teuthology.orchestra.run.smithi083.stdout: 5 73580 11.22 MB/sec execute 5 sec latency 32.700 ms 2024-04-15T21:37:38.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:37 smithi139 ceph-mon[26919]: pgmap v301: 97 pgs: 97 active+clean; 333 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 6.8 MiB/s wr, 651 op/s 2024-04-15T21:37:38.897 INFO:teuthology.orchestra.run.smithi083.stdout: 5 74324 14.57 MB/sec execute 6 sec latency 36.075 ms 2024-04-15T21:37:39.062 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-04-15T21:37:39.062 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-04-15T21:37:39.063 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-15T21:37:39.063 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-15T21:37:39.063 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-04-15T21:37:39.063 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-15T21:37:39.063 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-15T21:37:39.063 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-04-15T21:37:39.063 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-15T21:37:39.063 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-15T21:37:39.063 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-04-15T21:37:39.063 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-15T21:37:39.063 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-15T21:37:39.063 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-04-15T21:37:39.063 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-15T21:37:39.063 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-04-15T21:37:39.063 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-04-15T21:37:39.865 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:39 smithi083 ceph-mon[24790]: pgmap v302: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 9.3 MiB/s wr, 973 op/s 2024-04-15T21:37:39.865 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:39 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/533858402' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-15T21:37:39.897 INFO:teuthology.orchestra.run.smithi083.stdout: 5 74896 15.13 MB/sec execute 7 sec latency 27.225 ms 2024-04-15T21:37:40.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:39 smithi139 ceph-mon[26919]: pgmap v302: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 9.3 MiB/s wr, 973 op/s 2024-04-15T21:37:40.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:39 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/533858402' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-15T21:37:40.149 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -e sha1=a9a752df26c63acad72e1b3569fd79a515ca0765 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-04-15T21:37:40.897 INFO:teuthology.orchestra.run.smithi083.stdout: 5 75208 13.74 MB/sec execute 8 sec latency 34.679 ms 2024-04-15T21:37:41.893 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:41 smithi083 ceph-mon[24790]: pgmap v303: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 6.8 MiB/s wr, 728 op/s 2024-04-15T21:37:41.897 INFO:teuthology.orchestra.run.smithi083.stdout: 5 75596 12.95 MB/sec execute 9 sec latency 41.944 ms 2024-04-15T21:37:42.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:41 smithi139 ceph-mon[26919]: pgmap v303: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 6.8 MiB/s wr, 728 op/s 2024-04-15T21:37:42.898 INFO:teuthology.orchestra.run.smithi083.stdout: 5 76118 12.80 MB/sec execute 10 sec latency 36.019 ms 2024-04-15T21:37:43.872 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-04-15T21:37:43.897 INFO:teuthology.orchestra.run.smithi083.stdout: 5 76689 12.66 MB/sec execute 11 sec latency 31.731 ms 2024-04-15T21:37:43.924 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:43 smithi083 ceph-mon[24790]: pgmap v304: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 8.2 MiB/s wr, 811 op/s 2024-04-15T21:37:44.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:43 smithi139 ceph-mon[26919]: pgmap v304: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 8.2 MiB/s wr, 811 op/s 2024-04-15T21:37:44.898 INFO:teuthology.orchestra.run.smithi083.stdout: 5 77298 13.08 MB/sec execute 12 sec latency 33.718 ms 2024-04-15T21:37:44.929 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:44 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1932206455' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-15T21:37:44.986 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -e sha1=a9a752df26c63acad72e1b3569fd79a515ca0765 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-04-15T21:37:45.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:44 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/1932206455' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-15T21:37:45.898 INFO:teuthology.orchestra.run.smithi083.stdout: 5 77899 14.14 MB/sec execute 13 sec latency 30.167 ms 2024-04-15T21:37:45.919 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:45 smithi083 ceph-mon[24790]: pgmap v305: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 8.0 MiB/s wr, 803 op/s 2024-04-15T21:37:46.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:45 smithi139 ceph-mon[26919]: pgmap v305: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 8.0 MiB/s wr, 803 op/s 2024-04-15T21:37:46.898 INFO:teuthology.orchestra.run.smithi083.stdout: 5 78476 14.41 MB/sec execute 14 sec latency 23.748 ms 2024-04-15T21:37:47.898 INFO:teuthology.orchestra.run.smithi083.stdout: 5 78943 13.87 MB/sec execute 15 sec latency 47.858 ms 2024-04-15T21:37:47.933 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:47 smithi083 ceph-mon[24790]: pgmap v306: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.3 MiB/s wr, 727 op/s 2024-04-15T21:37:48.113 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:47 smithi139 ceph-mon[26919]: pgmap v306: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.3 MiB/s wr, 727 op/s 2024-04-15T21:37:48.898 INFO:teuthology.orchestra.run.smithi083.stdout: 5 79500 13.70 MB/sec execute 16 sec latency 24.568 ms 2024-04-15T21:37:49.898 INFO:teuthology.orchestra.run.smithi083.stdout: 5 80075 13.75 MB/sec execute 17 sec latency 33.889 ms 2024-04-15T21:37:49.950 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:49 smithi083 ceph-mon[24790]: pgmap v307: 97 pgs: 97 active+clean; 321 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 9.2 MiB/s wr, 844 op/s 2024-04-15T21:37:49.950 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:49 smithi083 ceph-mon[24790]: from='client.? 172.21.15.83:0/1306808054' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-15T21:37:49.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-15T21:37:49.999 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_461a84e6fbbf089011eafd13b98169fb9e9b8cc9/qa/tasks/cephadm.py", line 1364, in shell _shell(ctx, cluster_name, remote, File "/home/teuthworker/src/git.ceph.com_ceph_461a84e6fbbf089011eafd13b98169fb9e9b8cc9/qa/tasks/cephadm.py", line 39, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi083 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -e sha1=a9a752df26c63acad72e1b3569fd79a515ca0765 -- bash -c \'ceph versions | jq -e \'"\'"\'.overall | keys\'"\'"\' | grep $sha1\'' 2024-04-15T21:37:50.224 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=fcf64b3f13ce49289de702f7351883a1 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_461a84e6fbbf089011eafd13b98169fb9e9b8cc9/qa/tasks/cephadm.py", line 1364, in shell _shell(ctx, cluster_name, remote, File "/home/teuthworker/src/git.ceph.com_ceph_461a84e6fbbf089011eafd13b98169fb9e9b8cc9/qa/tasks/cephadm.py", line 39, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi083 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -e sha1=a9a752df26c63acad72e1b3569fd79a515ca0765 -- bash -c \'ceph versions | jq -e \'"\'"\'.overall | keys\'"\'"\' | grep $sha1\'' 2024-04-15T21:37:50.227 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-15T21:37:50.237 INFO:tasks.cephadm:Teardown begin 2024-04-15T21:37:50.238 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-15T21:37:50.246 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:49 smithi139 ceph-mon[26919]: pgmap v307: 97 pgs: 97 active+clean; 321 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 9.2 MiB/s wr, 844 op/s 2024-04-15T21:37:50.246 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:49 smithi139 ceph-mon[26919]: from='client.? 172.21.15.83:0/1306808054' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-15T21:37:50.288 DEBUG:teuthology.orchestra.run.smithi139:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-15T21:37:50.326 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-15T21:37:50.327 DEBUG:teuthology.orchestra.run.smithi083:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-15T21:37:50.348 DEBUG:teuthology.orchestra.run.smithi139:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-15T21:37:50.385 INFO:tasks.cephadm:Stopping all daemons... 2024-04-15T21:37:50.385 INFO:tasks.cephadm.mon.smithi083:Stopping mon.smithi083... 2024-04-15T21:37:50.386 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@mon.smithi083 2024-04-15T21:37:50.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:50 smithi083 systemd[1]: Stopping Ceph mon.smithi083 for e4a42038-fb6e-11ee-bc8f-c7b262605968... 2024-04-15T21:37:50.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:50 smithi083 bash[57724]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon.smithi083" found: no such container 2024-04-15T21:37:50.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:50 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi083[24785]: 2024-04-15T21:37:50.649+0000 7f03cb27a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi083 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-15T21:37:50.705 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:50 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi083[24785]: 2024-04-15T21:37:50.649+0000 7f03cb27a700 -1 mon.smithi083@0(leader) e2 *** Got Signal Terminated *** 2024-04-15T21:37:50.705 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:50 smithi083 podman[57733]: 2024-04-15 21:37:50.671538976 +0000 UTC m=+0.160845368 container died 4e1e2c99fe19bd8fa729652cdf80ec9ef25ddf983c1e094e9d33858a1bda80e0 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi083, RELEASE=HEAD, GIT_CLEAN=True, name=centos-stream, ceph=True, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8) 2024-04-15T21:37:50.898 INFO:teuthology.orchestra.run.smithi083.stdout: 5 80781 13.98 MB/sec execute 18 sec latency 27.697 ms 2024-04-15T21:37:51.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:51 smithi083 podman[57733]: 2024-04-15 21:37:51.391326359 +0000 UTC m=+0.880632755 container cleanup 4e1e2c99fe19bd8fa729652cdf80ec9ef25ddf983c1e094e9d33858a1bda80e0 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi083, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, io.openshift.expose-services=, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, name=centos-stream, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, com.redhat.component=centos-stream-container, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754) 2024-04-15T21:37:51.458 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:51 smithi083 bash[57733]: ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi083 2024-04-15T21:37:51.458 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Apr 15 21:37:51 smithi083 bash[57757]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon.smithi083" found: no such container 2024-04-15T21:37:51.898 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81428 14.70 MB/sec execute 19 sec latency 29.424 ms 2024-04-15T21:37:51.988 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@mon.smithi083.service' 2024-04-15T21:37:52.050 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-15T21:37:52.050 INFO:tasks.cephadm.mon.smithi083:Stopped mon.smithi083 2024-04-15T21:37:52.050 INFO:tasks.cephadm.mon.smithi139:Stopping mon.smithi139... 2024-04-15T21:37:52.051 DEBUG:teuthology.orchestra.run.smithi139:> sudo systemctl stop ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@mon.smithi139 2024-04-15T21:37:52.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:52 smithi139 systemd[1]: Stopping Ceph mon.smithi139 for e4a42038-fb6e-11ee-bc8f-c7b262605968... 2024-04-15T21:37:52.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:52 smithi139 bash[43063]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon.smithi139" found: no such container 2024-04-15T21:37:52.899 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81839 14.62 MB/sec execute 20 sec latency 57.424 ms 2024-04-15T21:37:53.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:53 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi139[26913]: 2024-04-15T21:37:53.067+0000 7f27b2011700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi139 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-15T21:37:53.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:53 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi139[26913]: 2024-04-15T21:37:53.067+0000 7f27b2011700 -1 mon.smithi139@1(peon) e2 *** Got Signal Terminated *** 2024-04-15T21:37:53.363 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:53 smithi139 podman[43076]: 2024-04-15 21:37:53.139673846 +0000 UTC m=+1.008170391 container died 3e6950ee7383f3fb51a25a3a89a74ff68994598286712be292d7b2d6d065f996 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi139, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, distribution-scope=public, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.openshift.expose-services=, ceph=True, release=754, maintainer=Guillaume Abrioux , architecture=x86_64) 2024-04-15T21:37:53.682 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:53 smithi139 podman[43076]: 2024-04-15 21:37:53.643678062 +0000 UTC m=+1.512174648 container cleanup 3e6950ee7383f3fb51a25a3a89a74ff68994598286712be292d7b2d6d065f996 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi139, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , architecture=x86_64, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, vcs-type=git, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-15T21:37:53.683 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:53 smithi139 bash[43076]: ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon-smithi139 2024-04-15T21:37:53.683 INFO:journalctl@ceph.mon.smithi139.smithi139.stdout:Apr 15 21:37:53 smithi139 bash[43102]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-mon.smithi139" found: no such container 2024-04-15T21:37:53.898 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 14.13 MB/sec execute 21 sec latency 828.489 ms 2024-04-15T21:37:53.905 DEBUG:teuthology.orchestra.run.smithi139:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@mon.smithi139.service' 2024-04-15T21:37:53.952 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-15T21:37:53.952 INFO:tasks.cephadm.mon.smithi139:Stopped mon.smithi139 2024-04-15T21:37:53.952 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-15T21:37:53.952 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.0 2024-04-15T21:37:54.351 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:37:54 smithi083 systemd[1]: Stopping Ceph osd.0 for e4a42038-fb6e-11ee-bc8f-c7b262605968... 2024-04-15T21:37:54.351 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:37:54 smithi083 bash[57807]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd.0" found: no such container 2024-04-15T21:37:54.351 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:37:54 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-0[34902]: 2024-04-15T21:37:54.162+0000 7f731654d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-15T21:37:54.351 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:37:54 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-0[34902]: 2024-04-15T21:37:54.162+0000 7f731654d700 -1 osd.0 69 *** Got signal Terminated *** 2024-04-15T21:37:54.352 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:37:54 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-0[34902]: 2024-04-15T21:37:54.162+0000 7f731654d700 -1 osd.0 69 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-15T21:37:54.898 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 13.49 MB/sec execute 22 sec latency 1828.527 ms 2024-04-15T21:37:55.898 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 12.90 MB/sec execute 23 sec latency 2828.557 ms 2024-04-15T21:37:56.899 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 12.36 MB/sec execute 24 sec latency 3828.585 ms 2024-04-15T21:37:57.898 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 11.87 MB/sec execute 25 sec latency 4828.615 ms 2024-04-15T21:37:58.898 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 11.41 MB/sec execute 26 sec latency 5828.669 ms 2024-04-15T21:37:59.899 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 10.99 MB/sec execute 27 sec latency 6828.737 ms 2024-04-15T21:38:00.101 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:37:59 smithi083 podman[57820]: 2024-04-15 21:37:59.805890375 +0000 UTC m=+5.757367801 container died a5cc4573e142a09a136ab5ff8b5054baef74bcd18b093f92bde883f78b4a7f08 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, vcs-type=git, io.openshift.expose-services=, distribution-scope=public, version=8, release=754, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True) 2024-04-15T21:38:00.493 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:38:00 smithi083 podman[57820]: 2024-04-15 21:38:00.214800227 +0000 UTC m=+6.166277640 container cleanup a5cc4573e142a09a136ab5ff8b5054baef74bcd18b093f92bde883f78b4a7f08 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-0, version=8, build-date=2022-05-03T08:36:31.336870, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vcs-type=git, io.buildah.version=1.19.8, RELEASE=HEAD, release=754, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, io.openshift.expose-services=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0) 2024-04-15T21:38:00.494 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:38:00 smithi083 bash[57820]: ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-0 2024-04-15T21:38:00.494 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:38:00 smithi083 bash[57848]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd.0" found: no such container 2024-04-15T21:38:00.851 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:38:00 smithi083 podman[57835]: 2024-04-15 21:38:00.491227924 +0000 UTC m=+0.680669317 container remove a5cc4573e142a09a136ab5ff8b5054baef74bcd18b093f92bde883f78b4a7f08 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-0, build-date=2022-05-03T08:36:31.336870, name=centos-stream, GIT_CLEAN=True, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, version=8, io.openshift.expose-services=, RELEASE=HEAD, vcs-type=git, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-15T21:38:00.851 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:38:00 smithi083 podman[57909]: 2024-04-15 21:38:00.629777851 +0000 UTC m=+0.019661097 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-15T21:38:00.851 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:38:00 smithi083 podman[57909]: 2024-04-15 21:38:00.734070969 +0000 UTC m=+0.123954224 container create ca4dd06a02b53d29e9f982d25abd01dd244e102b87fd1379567575773106bc65 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-0-deactivate, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, version=8, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, ceph=True, vcs-type=git, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD) 2024-04-15T21:38:00.898 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 10.60 MB/sec execute 28 sec latency 7828.792 ms 2024-04-15T21:38:01.351 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:38:01 smithi083 podman[57909]: 2024-04-15 21:38:01.003502576 +0000 UTC m=+0.393385823 container init ca4dd06a02b53d29e9f982d25abd01dd244e102b87fd1379567575773106bc65 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, GIT_CLEAN=True, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, name=centos-stream, version=8, io.buildah.version=1.19.8, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8) 2024-04-15T21:38:01.351 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:38:01 smithi083 podman[57909]: 2024-04-15 21:38:01.010901709 +0000 UTC m=+0.400784963 container start ca4dd06a02b53d29e9f982d25abd01dd244e102b87fd1379567575773106bc65 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-0-deactivate, architecture=x86_64, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, name=centos-stream, release=754, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, vcs-type=git) 2024-04-15T21:38:01.352 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:38:01 smithi083 podman[57909]: 2024-04-15 21:38:01.095172528 +0000 UTC m=+0.485055785 container attach ca4dd06a02b53d29e9f982d25abd01dd244e102b87fd1379567575773106bc65 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-0-deactivate, com.redhat.component=centos-stream-container, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, RELEASE=HEAD, version=8, io.k8s.display-name=CentOS Stream 8, ceph=True, GIT_BRANCH=HEAD, name=centos-stream) 2024-04-15T21:38:01.725 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 15 21:38:01 smithi083 podman[57909]: 2024-04-15 21:38:01.361991415 +0000 UTC m=+0.751874654 container died ca4dd06a02b53d29e9f982d25abd01dd244e102b87fd1379567575773106bc65 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-0-deactivate, io.openshift.expose-services=, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, version=8, distribution-scope=public, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, release=754, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, ceph=True) 2024-04-15T21:38:01.898 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 10.23 MB/sec execute 29 sec latency 8828.815 ms 2024-04-15T21:38:02.065 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.0.service' 2024-04-15T21:38:02.106 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-15T21:38:02.107 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-15T21:38:02.107 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-15T21:38:02.107 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.1 2024-04-15T21:38:02.351 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:38:02 smithi083 systemd[1]: Stopping Ceph osd.1 for e4a42038-fb6e-11ee-bc8f-c7b262605968... 2024-04-15T21:38:02.351 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:38:02 smithi083 bash[57987]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd.1" found: no such container 2024-04-15T21:38:02.899 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 9.89 MB/sec execute 30 sec latency 9828.864 ms 2024-04-15T21:38:03.101 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:38:02 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-1[38036]: 2024-04-15T21:38:02.660+0000 7f7de0b75700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-15T21:38:03.101 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:38:02 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-1[38036]: 2024-04-15T21:38:02.660+0000 7f7de0b75700 -1 osd.1 69 *** Got signal Terminated *** 2024-04-15T21:38:03.101 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:38:02 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-1[38036]: 2024-04-15T21:38:02.660+0000 7f7de0b75700 -1 osd.1 69 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-15T21:38:03.899 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 9.57 MB/sec execute 31 sec latency 10828.914 ms 2024-04-15T21:38:04.899 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 9.27 MB/sec execute 32 sec latency 11828.964 ms 2024-04-15T21:38:05.899 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 8.99 MB/sec execute 33 sec latency 12829.013 ms 2024-04-15T21:38:06.899 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 8.73 MB/sec execute 34 sec latency 13829.068 ms 2024-04-15T21:38:07.899 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 8.48 MB/sec execute 35 sec latency 14829.095 ms 2024-04-15T21:38:08.493 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:38:08 smithi083 podman[57999]: 2024-04-15 21:38:08.161058946 +0000 UTC m=+5.915772148 container died 4a973ccf37308907de9f353bc1e2a597a011de4fbc1e4f5c16a462bb488a4f9b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-1, ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.k8s.display-name=CentOS Stream 8, name=centos-stream, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, vcs-type=git) 2024-04-15T21:38:08.772 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:38:08 smithi083 podman[57999]: 2024-04-15 21:38:08.594088369 +0000 UTC m=+6.348801569 container cleanup 4a973ccf37308907de9f353bc1e2a597a011de4fbc1e4f5c16a462bb488a4f9b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-1, architecture=x86_64, io.openshift.expose-services=, vendor=Red Hat, Inc., version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, release=754, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.k8s.display-name=CentOS Stream 8, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-15T21:38:08.772 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:38:08 smithi083 bash[57999]: ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-1 2024-04-15T21:38:08.899 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 8.24 MB/sec execute 36 sec latency 15829.149 ms 2024-04-15T21:38:09.043 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:38:08 smithi083 podman[58018]: 2024-04-15 21:38:08.770435607 +0000 UTC m=+0.612246000 container remove 4a973ccf37308907de9f353bc1e2a597a011de4fbc1e4f5c16a462bb488a4f9b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-1, io.openshift.expose-services=, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., architecture=x86_64, ceph=True, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-15T21:38:09.043 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:38:08 smithi083 bash[58031]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd.1" found: no such container 2024-04-15T21:38:09.301 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:38:09 smithi083 podman[58097]: 2024-04-15 21:38:08.944568189 +0000 UTC m=+0.018881496 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-15T21:38:09.301 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:38:09 smithi083 podman[58097]: 2024-04-15 21:38:09.063093173 +0000 UTC m=+0.137406482 container create a395686738526d0e85ef0fb3094376d9ed4bca53903784e303adc5de6edda178 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-1-deactivate, io.openshift.expose-services=, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_CLEAN=True, ceph=True, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, version=8, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream) 2024-04-15T21:38:09.301 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:38:09 smithi083 podman[58097]: 2024-04-15 21:38:09.298914186 +0000 UTC m=+0.373227501 container init a395686738526d0e85ef0fb3094376d9ed4bca53903784e303adc5de6edda178 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-1-deactivate, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, RELEASE=HEAD, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, vcs-type=git, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, release=754, name=centos-stream, io.openshift.expose-services=, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-15T21:38:09.601 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:38:09 smithi083 podman[58097]: 2024-04-15 21:38:09.303659253 +0000 UTC m=+0.377972565 container start a395686738526d0e85ef0fb3094376d9ed4bca53903784e303adc5de6edda178 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-1-deactivate, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, release=754, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, com.redhat.component=centos-stream-container, name=centos-stream, GIT_CLEAN=True, ceph=True) 2024-04-15T21:38:09.601 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:38:09 smithi083 podman[58097]: 2024-04-15 21:38:09.399010749 +0000 UTC m=+0.473324060 container attach a395686738526d0e85ef0fb3094376d9ed4bca53903784e303adc5de6edda178 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-1-deactivate, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.buildah.version=1.19.8, architecture=x86_64, vcs-type=git, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.openshift.expose-services=, release=754, name=centos-stream, version=8, distribution-scope=public, vendor=Red Hat, Inc.) 2024-04-15T21:38:09.899 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 8.02 MB/sec execute 37 sec latency 16829.203 ms 2024-04-15T21:38:09.913 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 15 21:38:09 smithi083 podman[58097]: 2024-04-15 21:38:09.654345611 +0000 UTC m=+0.728658920 container died a395686738526d0e85ef0fb3094376d9ed4bca53903784e303adc5de6edda178 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-1-deactivate, vcs-type=git, GIT_BRANCH=HEAD, name=centos-stream, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, ceph=True, io.buildah.version=1.19.8, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, architecture=x86_64, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, release=754, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8) 2024-04-15T21:38:10.350 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.1.service' 2024-04-15T21:38:10.407 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-15T21:38:10.407 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-15T21:38:10.408 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-15T21:38:10.408 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.2 2024-04-15T21:38:10.576 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:38:10 smithi083 systemd[1]: Stopping Ceph osd.2 for e4a42038-fb6e-11ee-bc8f-c7b262605968... 2024-04-15T21:38:10.851 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:38:10 smithi083 bash[58175]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd.2" found: no such container 2024-04-15T21:38:10.851 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:38:10 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-2[41888]: 2024-04-15T21:38:10.702+0000 7f1e435b4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-15T21:38:10.851 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:38:10 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-2[41888]: 2024-04-15T21:38:10.702+0000 7f1e435b4700 -1 osd.2 69 *** Got signal Terminated *** 2024-04-15T21:38:10.851 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:38:10 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-2[41888]: 2024-04-15T21:38:10.702+0000 7f1e435b4700 -1 osd.2 69 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-15T21:38:10.899 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 7.81 MB/sec execute 38 sec latency 17829.254 ms 2024-04-15T21:38:11.899 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 7.61 MB/sec execute 39 sec latency 18829.303 ms 2024-04-15T21:38:12.899 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 7.42 MB/sec execute 40 sec latency 19829.357 ms 2024-04-15T21:38:13.899 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 7.24 MB/sec execute 41 sec latency 20829.383 ms 2024-04-15T21:38:14.899 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 7.06 MB/sec execute 42 sec latency 21829.433 ms 2024-04-15T21:38:15.899 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 6.90 MB/sec execute 43 sec latency 22829.460 ms 2024-04-15T21:38:16.592 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:38:16 smithi083 podman[58188]: 2024-04-15 21:38:16.315296433 +0000 UTC m=+5.732850467 container died e1038d2e74622bddff56c4a7789cc7e75e96668239527fa6e62918e1278e7083 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-2, io.buildah.version=1.19.8, vendor=Red Hat, Inc., vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, ceph=True, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, name=centos-stream, release=754, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-15T21:38:16.851 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:38:16 smithi083 podman[58188]: 2024-04-15 21:38:16.681033843 +0000 UTC m=+6.098587883 container cleanup e1038d2e74622bddff56c4a7789cc7e75e96668239527fa6e62918e1278e7083 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-2, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, release=754, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8) 2024-04-15T21:38:16.851 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:38:16 smithi083 bash[58188]: ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-2 2024-04-15T21:38:16.851 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:38:16 smithi083 bash[58220]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd.2" found: no such container 2024-04-15T21:38:16.899 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 6.74 MB/sec execute 44 sec latency 23829.485 ms 2024-04-15T21:38:17.348 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:38:16 smithi083 podman[58207]: 2024-04-15 21:38:16.974628576 +0000 UTC m=+0.657275357 container remove e1038d2e74622bddff56c4a7789cc7e75e96668239527fa6e62918e1278e7083 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-2, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vcs-type=git, vendor=Red Hat, Inc., release=754, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=) 2024-04-15T21:38:17.348 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:38:17 smithi083 podman[58284]: 2024-04-15 21:38:17.092493006 +0000 UTC m=+0.018306215 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-15T21:38:17.348 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:38:17 smithi083 podman[58284]: 2024-04-15 21:38:17.20052317 +0000 UTC m=+0.126336361 container create 5d43872b5411085dc0f6a8b9ab5b7b5ee46d5aaac7a89fee5131884bd1c43bae (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-2-deactivate, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, distribution-scope=public, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , release=754, CEPH_POINT_RELEASE=-17.2.0, version=8, vcs-type=git, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-15T21:38:17.601 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:38:17 smithi083 podman[58284]: 2024-04-15 21:38:17.45299489 +0000 UTC m=+0.378808081 container init 5d43872b5411085dc0f6a8b9ab5b7b5ee46d5aaac7a89fee5131884bd1c43bae (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-2-deactivate, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, architecture=x86_64, name=centos-stream, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , distribution-scope=public, com.redhat.component=centos-stream-container, version=8, GIT_BRANCH=HEAD, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-15T21:38:17.601 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:38:17 smithi083 podman[58284]: 2024-04-15 21:38:17.463508641 +0000 UTC m=+0.389321845 container start 5d43872b5411085dc0f6a8b9ab5b7b5ee46d5aaac7a89fee5131884bd1c43bae (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-2-deactivate, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.19.8, io.openshift.expose-services=, RELEASE=HEAD, architecture=x86_64, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-15T21:38:17.601 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:38:17 smithi083 podman[58284]: 2024-04-15 21:38:17.544631401 +0000 UTC m=+0.470444599 container attach 5d43872b5411085dc0f6a8b9ab5b7b5ee46d5aaac7a89fee5131884bd1c43bae (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, distribution-scope=public, version=8, release=754, io.buildah.version=1.19.8, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, name=centos-stream, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True) 2024-04-15T21:38:17.899 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 6.59 MB/sec execute 45 sec latency 24829.524 ms 2024-04-15T21:38:18.101 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 15 21:38:17 smithi083 podman[58284]: 2024-04-15 21:38:17.821932419 +0000 UTC m=+0.747745605 container died 5d43872b5411085dc0f6a8b9ab5b7b5ee46d5aaac7a89fee5131884bd1c43bae (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-2-deactivate, io.openshift.expose-services=, release=754, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , ceph=True, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, vendor=Red Hat, Inc., GIT_CLEAN=True, version=8, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container) 2024-04-15T21:38:18.471 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.2.service' 2024-04-15T21:38:18.527 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-15T21:38:18.527 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-15T21:38:18.527 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-15T21:38:18.527 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.3 2024-04-15T21:38:18.692 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:38:18 smithi083 systemd[1]: Stopping Ceph osd.3 for e4a42038-fb6e-11ee-bc8f-c7b262605968... 2024-04-15T21:38:18.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 6.45 MB/sec execute 46 sec latency 25829.576 ms 2024-04-15T21:38:19.101 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:38:18 smithi083 bash[58363]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd.3" found: no such container 2024-04-15T21:38:19.101 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:38:18 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-3[45745]: 2024-04-15T21:38:18.806+0000 7f8a0b75d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-15T21:38:19.101 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:38:18 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-3[45745]: 2024-04-15T21:38:18.806+0000 7f8a0b75d700 -1 osd.3 69 *** Got signal Terminated *** 2024-04-15T21:38:19.101 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:38:18 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-3[45745]: 2024-04-15T21:38:18.806+0000 7f8a0b75d700 -1 osd.3 69 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-15T21:38:19.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 6.31 MB/sec execute 47 sec latency 26829.808 ms 2024-04-15T21:38:20.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 6.18 MB/sec execute 48 sec latency 27829.877 ms 2024-04-15T21:38:21.363 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:21 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4[31176]: 2024-04-15T21:38:21.157+0000 7f252c8ef700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:54.363381+0000 front 2024-04-15T21:37:54.363466+0000 (oldest deadline 2024-04-15T21:38:20.763970+0000) 2024-04-15T21:38:21.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 6.05 MB/sec execute 49 sec latency 28829.924 ms 2024-04-15T21:38:22.351 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:38:22 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-3[45745]: 2024-04-15T21:38:22.071+0000 7f8a048f1700 -1 osd.3 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.170288+0000 front 2024-04-15T21:37:56.170276+0000 (oldest deadline 2024-04-15T21:38:21.470314+0000) 2024-04-15T21:38:22.433 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:22 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4[31176]: 2024-04-15T21:38:22.173+0000 7f252c8ef700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:54.363381+0000 front 2024-04-15T21:37:54.363466+0000 (oldest deadline 2024-04-15T21:38:20.763970+0000) 2024-04-15T21:38:22.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 5.93 MB/sec execute 50 sec latency 29829.951 ms 2024-04-15T21:38:23.351 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:38:23 smithi083 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-3[45745]: 2024-04-15T21:38:23.034+0000 7f8a048f1700 -1 osd.3 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.170288+0000 front 2024-04-15T21:37:56.170276+0000 (oldest deadline 2024-04-15T21:38:21.470314+0000) 2024-04-15T21:38:23.542 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:23 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4[31176]: 2024-04-15T21:38:23.192+0000 7f252c8ef700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:54.363381+0000 front 2024-04-15T21:37:54.363466+0000 (oldest deadline 2024-04-15T21:38:20.763970+0000) 2024-04-15T21:38:23.863 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:23 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:23.539+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:23.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 5.82 MB/sec execute 51 sec latency 30830.002 ms 2024-04-15T21:38:24.537 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:24 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4[31176]: 2024-04-15T21:38:24.166+0000 7f252c8ef700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:54.363381+0000 front 2024-04-15T21:37:54.363466+0000 (oldest deadline 2024-04-15T21:38:20.763970+0000) 2024-04-15T21:38:24.601 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:38:24 smithi083 podman[58376]: 2024-04-15 21:38:24.346050403 +0000 UTC m=+5.646528795 container died aad5a54fb42f45c0ed03369ed054dbfa3e0b8c11107f703e7cae2af4cbcc5052 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-3, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, RELEASE=HEAD, ceph=True, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, architecture=x86_64, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-15T21:38:24.866 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:24 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:24.534+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:24.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 5.71 MB/sec execute 52 sec latency 31830.028 ms 2024-04-15T21:38:24.987 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:38:24 smithi083 podman[58376]: 2024-04-15 21:38:24.734372622 +0000 UTC m=+6.034851019 container cleanup aad5a54fb42f45c0ed03369ed054dbfa3e0b8c11107f703e7cae2af4cbcc5052 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-3, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, release=754, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, distribution-scope=public, vcs-type=git, architecture=x86_64) 2024-04-15T21:38:24.987 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:38:24 smithi083 bash[58376]: ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-3 2024-04-15T21:38:24.987 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:38:24 smithi083 bash[58408]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd.3" found: no such container 2024-04-15T21:38:25.349 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:38:24 smithi083 podman[58396]: 2024-04-15 21:38:24.985540426 +0000 UTC m=+0.634302017 container remove aad5a54fb42f45c0ed03369ed054dbfa3e0b8c11107f703e7cae2af4cbcc5052 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-3, io.openshift.expose-services=, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, com.redhat.component=centos-stream-container, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, version=8, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-15T21:38:25.349 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:38:25 smithi083 podman[58470]: 2024-04-15 21:38:25.091311399 +0000 UTC m=+0.018686685 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-15T21:38:25.349 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:38:25 smithi083 podman[58470]: 2024-04-15 21:38:25.194895016 +0000 UTC m=+0.122270290 container create 28ea84d9de916d7334e9e7d6d1841ee0b821e710721b4357feef99b82f1f4719 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-3-deactivate, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, version=8, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-15T21:38:25.530 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:25 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4[31176]: 2024-04-15T21:38:25.144+0000 7f252c8ef700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:54.363381+0000 front 2024-04-15T21:37:54.363466+0000 (oldest deadline 2024-04-15T21:38:20.763970+0000) 2024-04-15T21:38:25.530 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:25 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:25.385+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.140702+0000 front 2024-04-15T21:37:58.140670+0000 (oldest deadline 2024-04-15T21:38:24.541560+0000) 2024-04-15T21:38:25.601 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:38:25 smithi083 podman[58470]: 2024-04-15 21:38:25.439072026 +0000 UTC m=+0.366447300 container init 28ea84d9de916d7334e9e7d6d1841ee0b821e710721b4357feef99b82f1f4719 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-3-deactivate, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, distribution-scope=public, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, release=754, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, RELEASE=HEAD, name=centos-stream, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux ) 2024-04-15T21:38:25.601 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:38:25 smithi083 podman[58470]: 2024-04-15 21:38:25.450139051 +0000 UTC m=+0.377514333 container start 28ea84d9de916d7334e9e7d6d1841ee0b821e710721b4357feef99b82f1f4719 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-3-deactivate, ceph=True, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, distribution-scope=public) 2024-04-15T21:38:25.601 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:38:25 smithi083 podman[58470]: 2024-04-15 21:38:25.522436873 +0000 UTC m=+0.449812148 container attach 28ea84d9de916d7334e9e7d6d1841ee0b821e710721b4357feef99b82f1f4719 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-3-deactivate, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, version=8, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, name=centos-stream, GIT_BRANCH=HEAD) 2024-04-15T21:38:25.863 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:25 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:25.527+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:25.863 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:25 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:25.754+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:25.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 5.60 MB/sec execute 53 sec latency 32830.059 ms 2024-04-15T21:38:26.094 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 15 21:38:25 smithi083 podman[58470]: 2024-04-15 21:38:25.785563386 +0000 UTC m=+0.712938651 container died 28ea84d9de916d7334e9e7d6d1841ee0b821e710721b4357feef99b82f1f4719 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-3-deactivate, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, architecture=x86_64, GIT_CLEAN=True, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, ceph=True, version=8, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream) 2024-04-15T21:38:26.434 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.3.service' 2024-04-15T21:38:26.484 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-15T21:38:26.484 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-15T21:38:26.484 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-15T21:38:26.485 DEBUG:teuthology.orchestra.run.smithi139:> sudo systemctl stop ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.4 2024-04-15T21:38:26.517 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:26 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4[31176]: 2024-04-15T21:38:26.162+0000 7f252c8ef700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:54.363381+0000 front 2024-04-15T21:37:54.363466+0000 (oldest deadline 2024-04-15T21:38:20.763970+0000) 2024-04-15T21:38:26.517 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:26 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:26.367+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.140702+0000 front 2024-04-15T21:37:58.140670+0000 (oldest deadline 2024-04-15T21:38:24.541560+0000) 2024-04-15T21:38:26.863 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:26 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:26.544+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:26.864 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:26 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:26.716+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:26.864 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:26 smithi139 systemd[1]: Stopping Ceph osd.4 for e4a42038-fb6e-11ee-bc8f-c7b262605968... 2024-04-15T21:38:26.864 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:26 smithi139 bash[43166]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd.4" found: no such container 2024-04-15T21:38:26.864 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:26 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4[31176]: 2024-04-15T21:38:26.677+0000 7f253375b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-15T21:38:26.864 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:26 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4[31176]: 2024-04-15T21:38:26.677+0000 7f253375b700 -1 osd.4 69 *** Got signal Terminated *** 2024-04-15T21:38:26.865 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:26 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4[31176]: 2024-04-15T21:38:26.677+0000 7f253375b700 -1 osd.4 69 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-15T21:38:26.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 5.49 MB/sec execute 54 sec latency 33830.086 ms 2024-04-15T21:38:27.521 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:27 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4[31176]: 2024-04-15T21:38:27.141+0000 7f252c8ef700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:54.363381+0000 front 2024-04-15T21:37:54.363466+0000 (oldest deadline 2024-04-15T21:38:20.763970+0000) 2024-04-15T21:38:27.521 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:27 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:27.321+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.140702+0000 front 2024-04-15T21:37:58.140670+0000 (oldest deadline 2024-04-15T21:38:24.541560+0000) 2024-04-15T21:38:27.863 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:27 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:27.518+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:27.863 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:27 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:27.693+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:27.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 5.39 MB/sec execute 55 sec latency 34830.148 ms 2024-04-15T21:38:28.508 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:28 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:28.309+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.140702+0000 front 2024-04-15T21:37:58.140670+0000 (oldest deadline 2024-04-15T21:38:24.541560+0000) 2024-04-15T21:38:28.509 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:28 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4[31176]: 2024-04-15T21:38:28.161+0000 7f252c8ef700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:54.363381+0000 front 2024-04-15T21:37:54.363466+0000 (oldest deadline 2024-04-15T21:38:20.763970+0000) 2024-04-15T21:38:28.863 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:28 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:28.506+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:28.863 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:28 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:28.697+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:28.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 5.30 MB/sec execute 56 sec latency 35830.200 ms 2024-04-15T21:38:29.467 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:29 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:29.355+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.140702+0000 front 2024-04-15T21:37:58.140670+0000 (oldest deadline 2024-04-15T21:38:24.541560+0000) 2024-04-15T21:38:29.468 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:29 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4[31176]: 2024-04-15T21:38:29.185+0000 7f252c8ef700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:54.363381+0000 front 2024-04-15T21:37:54.363466+0000 (oldest deadline 2024-04-15T21:38:20.763970+0000) 2024-04-15T21:38:29.468 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:29 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4[31176]: 2024-04-15T21:38:29.185+0000 7f252c8ef700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.271503+0000 front 2024-04-15T21:38:04.264876+0000 (oldest deadline 2024-04-15T21:38:28.365004+0000) 2024-04-15T21:38:29.733 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:29 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:29.464+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:29.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 5.21 MB/sec execute 57 sec latency 36830.273 ms 2024-04-15T21:38:30.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:29 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:29.730+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:30.613 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:30 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:30.312+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.140702+0000 front 2024-04-15T21:37:58.140670+0000 (oldest deadline 2024-04-15T21:38:24.541560+0000) 2024-04-15T21:38:30.613 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:30 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:30.439+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:30.614 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:30 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:30.439+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:03.887986+0000 front 2024-04-15T21:38:03.880697+0000 (oldest deadline 2024-04-15T21:38:29.780858+0000) 2024-04-15T21:38:30.614 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:30 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4[31176]: 2024-04-15T21:38:30.196+0000 7f252c8ef700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:54.363381+0000 front 2024-04-15T21:37:54.363466+0000 (oldest deadline 2024-04-15T21:38:20.763970+0000) 2024-04-15T21:38:30.614 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:30 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4[31176]: 2024-04-15T21:38:30.196+0000 7f252c8ef700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.271503+0000 front 2024-04-15T21:38:04.264876+0000 (oldest deadline 2024-04-15T21:38:28.365004+0000) 2024-04-15T21:38:30.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 5.12 MB/sec execute 58 sec latency 37830.340 ms 2024-04-15T21:38:31.097 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:30 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:30.770+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:31.097 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:30 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:30.770+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:31.363 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:31 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4[31176]: 2024-04-15T21:38:31.168+0000 7f252c8ef700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:54.363381+0000 front 2024-04-15T21:37:54.363466+0000 (oldest deadline 2024-04-15T21:38:20.763970+0000) 2024-04-15T21:38:31.363 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:31 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4[31176]: 2024-04-15T21:38:31.168+0000 7f252c8ef700 -1 osd.4 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.271503+0000 front 2024-04-15T21:38:04.264876+0000 (oldest deadline 2024-04-15T21:38:28.365004+0000) 2024-04-15T21:38:31.363 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:31 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:31.272+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.140702+0000 front 2024-04-15T21:37:58.140670+0000 (oldest deadline 2024-04-15T21:38:24.541560+0000) 2024-04-15T21:38:31.364 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:31 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:31.272+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:07.447824+0000 front 2024-04-15T21:38:07.442749+0000 (oldest deadline 2024-04-15T21:38:30.342734+0000) 2024-04-15T21:38:31.779 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:31 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:31.486+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:31.780 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:31 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:31.486+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:03.887986+0000 front 2024-04-15T21:38:03.880697+0000 (oldest deadline 2024-04-15T21:38:29.780858+0000) 2024-04-15T21:38:31.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 5.03 MB/sec execute 59 sec latency 38830.363 ms 2024-04-15T21:38:32.108 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:31 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:31.777+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:32.109 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:31 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:31.777+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:32.363 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:32 smithi139 podman[43177]: 2024-04-15 21:38:32.106586876 +0000 UTC m=+5.532737305 container died ca06e69e5c77a7c3f0003a93a8110630a57c11522e64d3d30af78f6ae34288e3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4, name=centos-stream, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, ceph=True, GIT_CLEAN=True, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, release=754, architecture=x86_64) 2024-04-15T21:38:32.364 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:32 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:32.307+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.140702+0000 front 2024-04-15T21:37:58.140670+0000 (oldest deadline 2024-04-15T21:38:24.541560+0000) 2024-04-15T21:38:32.364 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:32 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:32.307+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:07.447824+0000 front 2024-04-15T21:38:07.442749+0000 (oldest deadline 2024-04-15T21:38:30.342734+0000) 2024-04-15T21:38:32.750 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:32 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:32.437+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:32.750 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:32 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:32.437+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:03.887986+0000 front 2024-04-15T21:38:03.880697+0000 (oldest deadline 2024-04-15T21:38:29.780858+0000) 2024-04-15T21:38:32.750 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:32 smithi139 podman[43177]: 2024-04-15 21:38:32.5358407 +0000 UTC m=+5.961991126 container cleanup ca06e69e5c77a7c3f0003a93a8110630a57c11522e64d3d30af78f6ae34288e3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, ceph=True, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, name=centos-stream, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, release=754, GIT_CLEAN=True, vcs-type=git, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-15T21:38:32.750 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:32 smithi139 bash[43177]: ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4 2024-04-15T21:38:32.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 4.94 MB/sec execute 60 sec latency 39830.398 ms 2024-04-15T21:38:33.025 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:32 smithi139 podman[43197]: 2024-04-15 21:38:32.77041402 +0000 UTC m=+0.664416658 container remove ca06e69e5c77a7c3f0003a93a8110630a57c11522e64d3d30af78f6ae34288e3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, RELEASE=HEAD, ceph=True, com.redhat.component=centos-stream-container, version=8, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, GIT_CLEAN=True, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-15T21:38:33.025 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:32 smithi139 bash[43210]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd.4" found: no such container 2024-04-15T21:38:33.025 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:32 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:32.747+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:33.025 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:32 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:32.747+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:33.292 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:33 smithi139 podman[43274]: 2024-04-15 21:38:32.926119492 +0000 UTC m=+0.018178012 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-15T21:38:33.292 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:33 smithi139 podman[43274]: 2024-04-15 21:38:33.065574578 +0000 UTC m=+0.157633098 container create 7770e9f5e54fd03662097949a5813c516bd8a56b162fda38b3e79e259405c2ab (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4-deactivate, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, ceph=True, io.openshift.expose-services=, vendor=Red Hat, Inc., release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-15T21:38:33.292 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:33 smithi139 podman[43274]: 2024-04-15 21:38:33.290242178 +0000 UTC m=+0.382300713 container init 7770e9f5e54fd03662097949a5813c516bd8a56b162fda38b3e79e259405c2ab (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.openshift.expose-services=, architecture=x86_64, release=754, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_BRANCH=HEAD, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-15T21:38:33.582 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:33 smithi139 podman[43274]: 2024-04-15 21:38:33.293806117 +0000 UTC m=+0.385864646 container start 7770e9f5e54fd03662097949a5813c516bd8a56b162fda38b3e79e259405c2ab (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4-deactivate, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, architecture=x86_64, io.openshift.tags=base centos centos-stream, distribution-scope=public, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, GIT_BRANCH=HEAD, vcs-type=git, GIT_CLEAN=True, version=8, build-date=2022-05-03T08:36:31.336870, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-15T21:38:33.582 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:33 smithi139 podman[43274]: 2024-04-15 21:38:33.415380975 +0000 UTC m=+0.507439509 container attach 7770e9f5e54fd03662097949a5813c516bd8a56b162fda38b3e79e259405c2ab (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4-deactivate, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, architecture=x86_64, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, ceph=True, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, vcs-type=git) 2024-04-15T21:38:33.582 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:33 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:33.335+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.140702+0000 front 2024-04-15T21:37:58.140670+0000 (oldest deadline 2024-04-15T21:38:24.541560+0000) 2024-04-15T21:38:33.582 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:33 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:33.335+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:07.447824+0000 front 2024-04-15T21:38:07.442749+0000 (oldest deadline 2024-04-15T21:38:30.342734+0000) 2024-04-15T21:38:33.583 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:33 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:33.476+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:33.583 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:33 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:33.476+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:03.887986+0000 front 2024-04-15T21:38:03.880697+0000 (oldest deadline 2024-04-15T21:38:29.780858+0000) 2024-04-15T21:38:33.863 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:33 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:33.753+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:33.863 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:33 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:33.753+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:33.864 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 15 21:38:33 smithi139 podman[43274]: 2024-04-15 21:38:33.57983163 +0000 UTC m=+0.671890154 container died 7770e9f5e54fd03662097949a5813c516bd8a56b162fda38b3e79e259405c2ab (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-4-deactivate, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, vcs-type=git, io.openshift.expose-services=, version=8, io.buildah.version=1.19.8, architecture=x86_64, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-15T21:38:33.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 4.86 MB/sec execute 61 sec latency 40830.450 ms 2024-04-15T21:38:34.249 DEBUG:teuthology.orchestra.run.smithi139:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.4.service' 2024-04-15T21:38:34.334 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-15T21:38:34.334 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-15T21:38:34.334 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-15T21:38:34.335 DEBUG:teuthology.orchestra.run.smithi139:> sudo systemctl stop ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.5 2024-04-15T21:38:34.489 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:34 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:34.461+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:34.489 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:34 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:34.461+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:03.887986+0000 front 2024-04-15T21:38:03.880697+0000 (oldest deadline 2024-04-15T21:38:29.780858+0000) 2024-04-15T21:38:34.489 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:34 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:34.364+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.140702+0000 front 2024-04-15T21:37:58.140670+0000 (oldest deadline 2024-04-15T21:38:24.541560+0000) 2024-04-15T21:38:34.489 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:34 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:34.364+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:07.447824+0000 front 2024-04-15T21:38:07.442749+0000 (oldest deadline 2024-04-15T21:38:30.342734+0000) 2024-04-15T21:38:34.489 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:34 smithi139 systemd[1]: Stopping Ceph osd.5 for e4a42038-fb6e-11ee-bc8f-c7b262605968... 2024-04-15T21:38:34.490 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:34 smithi139 bash[43353]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd.5" found: no such container 2024-04-15T21:38:34.758 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:34 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:34.597+0000 7fb91794f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-15T21:38:34.758 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:34 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:34.597+0000 7fb91794f700 -1 osd.5 69 *** Got signal Terminated *** 2024-04-15T21:38:34.759 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:34 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:34.597+0000 7fb91794f700 -1 osd.5 69 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-15T21:38:34.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 4.79 MB/sec execute 62 sec latency 41830.484 ms 2024-04-15T21:38:35.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:34 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:34.755+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:35.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:34 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:34.755+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:35.613 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:35 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:35.500+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:35.613 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:35 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:35.500+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:03.887986+0000 front 2024-04-15T21:38:03.880697+0000 (oldest deadline 2024-04-15T21:38:29.780858+0000) 2024-04-15T21:38:35.613 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:35 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:35.318+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.140702+0000 front 2024-04-15T21:37:58.140670+0000 (oldest deadline 2024-04-15T21:38:24.541560+0000) 2024-04-15T21:38:35.614 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:35 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:35.318+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:07.447824+0000 front 2024-04-15T21:38:07.442749+0000 (oldest deadline 2024-04-15T21:38:30.342734+0000) 2024-04-15T21:38:35.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 4.71 MB/sec execute 63 sec latency 42830.541 ms 2024-04-15T21:38:36.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:35 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:35.774+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:36.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:35 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:35.774+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:36.613 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:36 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:36.482+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:36.613 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:36 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:36.482+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:03.887986+0000 front 2024-04-15T21:38:03.880697+0000 (oldest deadline 2024-04-15T21:38:29.780858+0000) 2024-04-15T21:38:36.613 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:36 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:36.357+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.140702+0000 front 2024-04-15T21:37:58.140670+0000 (oldest deadline 2024-04-15T21:38:24.541560+0000) 2024-04-15T21:38:36.614 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:36 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:36.357+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:07.447824+0000 front 2024-04-15T21:38:07.442749+0000 (oldest deadline 2024-04-15T21:38:30.342734+0000) 2024-04-15T21:38:36.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 4.64 MB/sec execute 64 sec latency 43830.590 ms 2024-04-15T21:38:37.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:36 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:36.740+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:37.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:36 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:36.740+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:37.730 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:37 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:37.481+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:37.730 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:37 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:37.481+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:03.887986+0000 front 2024-04-15T21:38:03.880697+0000 (oldest deadline 2024-04-15T21:38:29.780858+0000) 2024-04-15T21:38:37.730 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:37 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:37.365+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.140702+0000 front 2024-04-15T21:37:58.140670+0000 (oldest deadline 2024-04-15T21:38:24.541560+0000) 2024-04-15T21:38:37.730 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:37 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:37.365+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:07.447824+0000 front 2024-04-15T21:38:07.442749+0000 (oldest deadline 2024-04-15T21:38:30.342734+0000) 2024-04-15T21:38:37.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 4.56 MB/sec execute 65 sec latency 44830.632 ms 2024-04-15T21:38:38.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:37 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:37.727+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:38.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:37 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:37.727+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:38.613 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:38 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:38.483+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:38.613 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:38 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:38.483+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:03.887986+0000 front 2024-04-15T21:38:03.880697+0000 (oldest deadline 2024-04-15T21:38:29.780858+0000) 2024-04-15T21:38:38.613 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:38 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:38.323+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.140702+0000 front 2024-04-15T21:37:58.140670+0000 (oldest deadline 2024-04-15T21:38:24.541560+0000) 2024-04-15T21:38:38.614 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:38 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:38.323+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:07.447824+0000 front 2024-04-15T21:38:07.442749+0000 (oldest deadline 2024-04-15T21:38:30.342734+0000) 2024-04-15T21:38:38.900 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 4.50 MB/sec execute 66 sec latency 45830.678 ms 2024-04-15T21:38:39.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:38 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:38.764+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:39.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:38 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:38.764+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:39.613 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:39 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:39.514+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:39.613 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:39 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:39.514+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:03.887986+0000 front 2024-04-15T21:38:03.880697+0000 (oldest deadline 2024-04-15T21:38:29.780858+0000) 2024-04-15T21:38:39.614 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:39 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:39.286+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.140702+0000 front 2024-04-15T21:37:58.140670+0000 (oldest deadline 2024-04-15T21:38:24.541560+0000) 2024-04-15T21:38:39.614 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:39 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:39.286+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:07.447824+0000 front 2024-04-15T21:38:07.442749+0000 (oldest deadline 2024-04-15T21:38:30.342734+0000) 2024-04-15T21:38:39.614 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:39 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5[34164]: 2024-04-15T21:38:39.286+0000 7fb914aeb700 -1 osd.5 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:14.443550+0000 front 2024-04-15T21:38:14.443577+0000 (oldest deadline 2024-04-15T21:38:39.143697+0000) 2024-04-15T21:38:39.901 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 4.43 MB/sec execute 67 sec latency 46830.725 ms 2024-04-15T21:38:40.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:39 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:39.759+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:40.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:39 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:39.759+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:40.424 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:40 smithi139 podman[43365]: 2024-04-15 21:38:40.146935297 +0000 UTC m=+5.691009146 container died 9696759277a4521f20d6aef2d3452ebd0067fe929ddfa70f782e146f31316985 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5, version=8, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, name=centos-stream, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, release=754, io.buildah.version=1.19.8, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.openshift.expose-services=) 2024-04-15T21:38:40.748 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:40 smithi139 podman[43365]: 2024-04-15 21:38:40.621490701 +0000 UTC m=+6.165564538 container cleanup 9696759277a4521f20d6aef2d3452ebd0067fe929ddfa70f782e146f31316985 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, io.openshift.expose-services=, vcs-type=git, io.k8s.display-name=CentOS Stream 8, version=8, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-15T21:38:40.748 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:40 smithi139 bash[43365]: ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5 2024-04-15T21:38:40.748 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:40 smithi139 bash[43407]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd.5" found: no such container 2024-04-15T21:38:40.748 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:40 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:40.490+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:40.749 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:40 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:40.490+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:03.887986+0000 front 2024-04-15T21:38:03.880697+0000 (oldest deadline 2024-04-15T21:38:29.780858+0000) 2024-04-15T21:38:40.901 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 4.36 MB/sec execute 68 sec latency 47830.769 ms 2024-04-15T21:38:41.096 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:40 smithi139 podman[43394]: 2024-04-15 21:38:40.906167356 +0000 UTC m=+0.753649884 container remove 9696759277a4521f20d6aef2d3452ebd0067fe929ddfa70f782e146f31316985 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, name=centos-stream, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, vcs-type=git, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., release=754) 2024-04-15T21:38:41.096 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:40 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:40.744+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:41.096 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:40 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:40.744+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:41.096 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:40 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:40.744+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:14.113428+0000 front 2024-04-15T21:38:14.111353+0000 (oldest deadline 2024-04-15T21:38:40.011447+0000) 2024-04-15T21:38:41.352 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:41 smithi139 podman[43470]: 2024-04-15 21:38:40.996960537 +0000 UTC m=+0.018781044 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-15T21:38:41.353 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:41 smithi139 podman[43470]: 2024-04-15 21:38:41.118321935 +0000 UTC m=+0.140142422 container create c520d498c773e401939ed2f33f17ce0e16cad96e15d9ae6f613006e31fa73ba8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5-deactivate, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, version=8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vcs-type=git, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, release=754, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, GIT_CLEAN=True, io.buildah.version=1.19.8, name=centos-stream, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc.) 2024-04-15T21:38:41.353 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:41 smithi139 podman[43470]: 2024-04-15 21:38:41.350747209 +0000 UTC m=+0.372567688 container init c520d498c773e401939ed2f33f17ce0e16cad96e15d9ae6f613006e31fa73ba8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5-deactivate, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., architecture=x86_64, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.openshift.expose-services=, RELEASE=HEAD, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream) 2024-04-15T21:38:41.613 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:41 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:41.511+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:41.613 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:41 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:41.511+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:03.887986+0000 front 2024-04-15T21:38:03.880697+0000 (oldest deadline 2024-04-15T21:38:29.780858+0000) 2024-04-15T21:38:41.613 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:41 smithi139 podman[43470]: 2024-04-15 21:38:41.354199225 +0000 UTC m=+0.376019705 container start c520d498c773e401939ed2f33f17ce0e16cad96e15d9ae6f613006e31fa73ba8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5-deactivate, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, RELEASE=HEAD, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, release=754, version=8, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , distribution-scope=public, vendor=Red Hat, Inc., ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container) 2024-04-15T21:38:41.614 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:41 smithi139 podman[43470]: 2024-04-15 21:38:41.442352436 +0000 UTC m=+0.464172931 container attach c520d498c773e401939ed2f33f17ce0e16cad96e15d9ae6f613006e31fa73ba8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5-deactivate, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, version=8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870) 2024-04-15T21:38:41.901 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 4.30 MB/sec execute 69 sec latency 48830.819 ms 2024-04-15T21:38:41.922 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 15 21:38:41 smithi139 podman[43470]: 2024-04-15 21:38:41.631797954 +0000 UTC m=+0.653618444 container died c520d498c773e401939ed2f33f17ce0e16cad96e15d9ae6f613006e31fa73ba8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, release=754, RELEASE=HEAD, version=8, io.openshift.tags=base centos centos-stream, name=centos-stream, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.openshift.expose-services=, vcs-type=git, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0) 2024-04-15T21:38:41.922 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:41 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:41.728+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:41.922 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:41 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:41.728+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:41.923 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:41 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:41.728+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:14.113428+0000 front 2024-04-15T21:38:14.111353+0000 (oldest deadline 2024-04-15T21:38:40.011447+0000) 2024-04-15T21:38:42.214 DEBUG:teuthology.orchestra.run.smithi139:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.5.service' 2024-04-15T21:38:42.257 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-15T21:38:42.258 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-15T21:38:42.258 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-15T21:38:42.258 DEBUG:teuthology.orchestra.run.smithi139:> sudo systemctl stop ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.6 2024-04-15T21:38:42.493 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:42 smithi139 systemd[1]: Stopping Ceph osd.6 for e4a42038-fb6e-11ee-bc8f-c7b262605968... 2024-04-15T21:38:42.493 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:42 smithi139 bash[43548]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd.6" found: no such container 2024-04-15T21:38:42.493 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:42 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:42.490+0000 7f78f45b1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-15T21:38:42.493 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:42 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:42.490+0000 7f78f45b1700 -1 osd.6 69 *** Got signal Terminated *** 2024-04-15T21:38:42.493 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:42 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:42.490+0000 7f78f45b1700 -1 osd.6 69 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-15T21:38:42.863 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:42 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:42.498+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:42.863 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:42 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:42.498+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:03.887986+0000 front 2024-04-15T21:38:03.880697+0000 (oldest deadline 2024-04-15T21:38:29.780858+0000) 2024-04-15T21:38:42.863 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:42 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:42.498+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:15.682072+0000 front 2024-04-15T21:38:15.682026+0000 (oldest deadline 2024-04-15T21:38:41.582135+0000) 2024-04-15T21:38:42.864 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:42 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:42.684+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:42.864 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:42 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:42.684+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:42.864 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:42 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:42.684+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:14.113428+0000 front 2024-04-15T21:38:14.111353+0000 (oldest deadline 2024-04-15T21:38:40.011447+0000) 2024-04-15T21:38:42.901 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 4.24 MB/sec execute 70 sec latency 49830.879 ms 2024-04-15T21:38:43.863 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:43 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:43.525+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:43.864 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:43 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:43.525+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:03.887986+0000 front 2024-04-15T21:38:03.880697+0000 (oldest deadline 2024-04-15T21:38:29.780858+0000) 2024-04-15T21:38:43.864 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:43 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:43.525+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:15.682072+0000 front 2024-04-15T21:38:15.682026+0000 (oldest deadline 2024-04-15T21:38:41.582135+0000) 2024-04-15T21:38:43.864 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:43 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:43.697+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:43.864 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:43 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:43.697+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:43.864 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:43 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:43.697+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:14.113428+0000 front 2024-04-15T21:38:14.111353+0000 (oldest deadline 2024-04-15T21:38:40.011447+0000) 2024-04-15T21:38:43.901 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 4.18 MB/sec execute 71 sec latency 50830.953 ms 2024-04-15T21:38:44.863 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:44 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:44.563+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:44.863 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:44 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:44.563+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:03.887986+0000 front 2024-04-15T21:38:03.880697+0000 (oldest deadline 2024-04-15T21:38:29.780858+0000) 2024-04-15T21:38:44.863 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:44 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:44.563+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:15.682072+0000 front 2024-04-15T21:38:15.682026+0000 (oldest deadline 2024-04-15T21:38:41.582135+0000) 2024-04-15T21:38:44.864 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:44 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:44.708+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:44.864 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:44 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:44.708+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:44.864 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:44 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:44.708+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:14.113428+0000 front 2024-04-15T21:38:14.111353+0000 (oldest deadline 2024-04-15T21:38:40.011447+0000) 2024-04-15T21:38:44.901 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 4.12 MB/sec execute 72 sec latency 51831.000 ms 2024-04-15T21:38:45.757 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:45 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:45.570+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:45.758 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:45 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:45.570+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:03.887986+0000 front 2024-04-15T21:38:03.880697+0000 (oldest deadline 2024-04-15T21:38:29.780858+0000) 2024-04-15T21:38:45.758 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:45 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:45.570+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:15.682072+0000 front 2024-04-15T21:38:15.682026+0000 (oldest deadline 2024-04-15T21:38:41.582135+0000) 2024-04-15T21:38:45.901 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 4.06 MB/sec execute 73 sec latency 52831.063 ms 2024-04-15T21:38:46.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:45 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:45.755+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:46.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:45 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:45.755+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:46.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:45 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:45.755+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:14.113428+0000 front 2024-04-15T21:38:14.111353+0000 (oldest deadline 2024-04-15T21:38:40.011447+0000) 2024-04-15T21:38:46.901 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 4.01 MB/sec execute 74 sec latency 53831.109 ms 2024-04-15T21:38:47.113 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:46 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:46.620+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:56.879569+0000 front 2024-04-15T21:37:56.886392+0000 (oldest deadline 2024-04-15T21:38:22.779770+0000) 2024-04-15T21:38:47.113 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:46 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:46.620+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:03.887986+0000 front 2024-04-15T21:38:03.880697+0000 (oldest deadline 2024-04-15T21:38:29.780858+0000) 2024-04-15T21:38:47.113 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:46 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6[37143]: 2024-04-15T21:38:46.620+0000 7f78eff4a700 -1 osd.6 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:15.682072+0000 front 2024-04-15T21:38:15.682026+0000 (oldest deadline 2024-04-15T21:38:41.582135+0000) 2024-04-15T21:38:47.114 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:46 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:46.720+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:47.114 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:46 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:46.720+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:47.114 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:46 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:46.720+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:14.113428+0000 front 2024-04-15T21:38:14.111353+0000 (oldest deadline 2024-04-15T21:38:40.011447+0000) 2024-04-15T21:38:47.114 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:46 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:46.720+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-15T21:38:20.011879+0000 front 2024-04-15T21:38:20.011835+0000 (oldest deadline 2024-04-15T21:38:45.911950+0000) 2024-04-15T21:38:47.901 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.96 MB/sec execute 75 sec latency 54831.167 ms 2024-04-15T21:38:48.078 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:47 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:47.766+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:48.078 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:47 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:47.766+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:48.079 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:47 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:47.766+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:14.113428+0000 front 2024-04-15T21:38:14.111353+0000 (oldest deadline 2024-04-15T21:38:40.011447+0000) 2024-04-15T21:38:48.079 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:47 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:47.766+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-15T21:38:20.011879+0000 front 2024-04-15T21:38:20.011835+0000 (oldest deadline 2024-04-15T21:38:45.911950+0000) 2024-04-15T21:38:48.351 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:48 smithi139 podman[43561]: 2024-04-15 21:38:48.076015309 +0000 UTC m=+5.705469971 container died 8108f3669964546e919d10a7a1649b4a691c17320730dfafd184069e5d4c04f9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, release=754, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, name=centos-stream, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, distribution-scope=public, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-15T21:38:48.613 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:48 smithi139 podman[43561]: 2024-04-15 21:38:48.456036616 +0000 UTC m=+6.085491280 container cleanup 8108f3669964546e919d10a7a1649b4a691c17320730dfafd184069e5d4c04f9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, release=754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, io.openshift.expose-services=, maintainer=Guillaume Abrioux , RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, architecture=x86_64, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream) 2024-04-15T21:38:48.614 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:48 smithi139 bash[43561]: ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6 2024-04-15T21:38:48.614 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:48 smithi139 bash[43591]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd.6" found: no such container 2024-04-15T21:38:48.901 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.90 MB/sec execute 76 sec latency 55831.236 ms 2024-04-15T21:38:49.094 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:48 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:48.736+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:49.094 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:48 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:48.736+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:49.094 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:48 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:48.736+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:14.113428+0000 front 2024-04-15T21:38:14.111353+0000 (oldest deadline 2024-04-15T21:38:40.011447+0000) 2024-04-15T21:38:49.095 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:48 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:48.736+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-15T21:38:20.011879+0000 front 2024-04-15T21:38:20.011835+0000 (oldest deadline 2024-04-15T21:38:45.911950+0000) 2024-04-15T21:38:49.095 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:48 smithi139 podman[43577]: 2024-04-15 21:38:48.748927647 +0000 UTC m=+0.669653682 container remove 8108f3669964546e919d10a7a1649b4a691c17320730dfafd184069e5d4c04f9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6, CEPH_POINT_RELEASE=-17.2.0, version=8, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, release=754, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., GIT_CLEAN=True, ceph=True, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-15T21:38:49.095 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:48 smithi139 podman[43656]: 2024-04-15 21:38:48.842750717 +0000 UTC m=+0.017988373 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-15T21:38:49.095 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:48 smithi139 podman[43656]: 2024-04-15 21:38:48.982932709 +0000 UTC m=+0.158170340 container create f3d943a02fb6ee802d2e477e4b0822c9b4fb1615f2d21f98cd36e2170879ca61 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6-deactivate, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, vcs-type=git, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, name=centos-stream, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, version=8, ceph=True) 2024-04-15T21:38:49.363 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:49 smithi139 podman[43656]: 2024-04-15 21:38:49.19358898 +0000 UTC m=+0.368826611 container init f3d943a02fb6ee802d2e477e4b0822c9b4fb1615f2d21f98cd36e2170879ca61 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6-deactivate, vcs-type=git, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-15T21:38:49.363 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:49 smithi139 podman[43656]: 2024-04-15 21:38:49.196847885 +0000 UTC m=+0.372085525 container start f3d943a02fb6ee802d2e477e4b0822c9b4fb1615f2d21f98cd36e2170879ca61 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6-deactivate, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_BRANCH=HEAD, release=754, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, architecture=x86_64, io.openshift.expose-services=, vcs-type=git, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True) 2024-04-15T21:38:49.363 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:49 smithi139 podman[43656]: 2024-04-15 21:38:49.276980218 +0000 UTC m=+0.452217858 container attach f3d943a02fb6ee802d2e477e4b0822c9b4fb1615f2d21f98cd36e2170879ca61 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6-deactivate, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.buildah.version=1.19.8, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.openshift.expose-services=, version=8, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, RELEASE=HEAD, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, io.openshift.tags=base centos centos-stream, vcs-type=git) 2024-04-15T21:38:49.748 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:49 smithi139 podman[43656]: 2024-04-15 21:38:49.479254417 +0000 UTC m=+0.654492059 container died f3d943a02fb6ee802d2e477e4b0822c9b4fb1615f2d21f98cd36e2170879ca61 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, name=centos-stream, RELEASE=HEAD, distribution-scope=public, GIT_BRANCH=HEAD, GIT_CLEAN=True, vcs-type=git, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-15T21:38:49.901 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.85 MB/sec execute 77 sec latency 56831.260 ms 2024-04-15T21:38:50.005 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 15 21:38:49 smithi139 podman[43682]: 2024-04-15 21:38:49.988143167 +0000 UTC m=+0.503535602 container remove f3d943a02fb6ee802d2e477e4b0822c9b4fb1615f2d21f98cd36e2170879ca61 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, ceph=True, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , release=754, com.redhat.component=centos-stream-container, vcs-type=git, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0) 2024-04-15T21:38:50.005 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:49 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:49.745+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:50.005 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:49 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:49.745+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:50.005 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:49 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:49.745+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:14.113428+0000 front 2024-04-15T21:38:14.111353+0000 (oldest deadline 2024-04-15T21:38:40.011447+0000) 2024-04-15T21:38:50.005 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:49 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:49.745+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-15T21:38:20.011879+0000 front 2024-04-15T21:38:20.011835+0000 (oldest deadline 2024-04-15T21:38:45.911950+0000) 2024-04-15T21:38:50.010 DEBUG:teuthology.orchestra.run.smithi139:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.6.service' 2024-04-15T21:38:50.051 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-15T21:38:50.052 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-15T21:38:50.052 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-15T21:38:50.052 DEBUG:teuthology.orchestra.run.smithi139:> sudo systemctl stop ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.7 2024-04-15T21:38:50.285 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:50 smithi139 systemd[1]: Stopping Ceph osd.7 for e4a42038-fb6e-11ee-bc8f-c7b262605968... 2024-04-15T21:38:50.285 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:50 smithi139 bash[43734]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd.7" found: no such container 2024-04-15T21:38:50.285 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:50 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:50.282+0000 7f64e5a0a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-15T21:38:50.286 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:50 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:50.282+0000 7f64e5a0a700 -1 osd.7 69 *** Got signal Terminated *** 2024-04-15T21:38:50.613 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:50 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:50.282+0000 7f64e5a0a700 -1 osd.7 69 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-15T21:38:50.901 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.80 MB/sec execute 78 sec latency 57831.326 ms 2024-04-15T21:38:51.097 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:50 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:50.706+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:51.097 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:50 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:50.706+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:51.097 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:50 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:50.706+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:14.113428+0000 front 2024-04-15T21:38:14.111353+0000 (oldest deadline 2024-04-15T21:38:40.011447+0000) 2024-04-15T21:38:51.098 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:50 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:50.706+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-15T21:38:20.011879+0000 front 2024-04-15T21:38:20.011835+0000 (oldest deadline 2024-04-15T21:38:45.911950+0000) 2024-04-15T21:38:51.901 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.76 MB/sec execute 79 sec latency 58831.378 ms 2024-04-15T21:38:52.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:51 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:51.725+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:52.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:51 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:51.725+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:52.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:51 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:51.725+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:14.113428+0000 front 2024-04-15T21:38:14.111353+0000 (oldest deadline 2024-04-15T21:38:40.011447+0000) 2024-04-15T21:38:52.114 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:51 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:51.725+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-15T21:38:20.011879+0000 front 2024-04-15T21:38:20.011835+0000 (oldest deadline 2024-04-15T21:38:45.911950+0000) 2024-04-15T21:38:52.863 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:52 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:52.721+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:52.863 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:52 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:52.721+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:52.863 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:52 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:52.721+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:14.113428+0000 front 2024-04-15T21:38:14.111353+0000 (oldest deadline 2024-04-15T21:38:40.011447+0000) 2024-04-15T21:38:52.863 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:52 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:52.721+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-15T21:38:20.011879+0000 front 2024-04-15T21:38:20.011835+0000 (oldest deadline 2024-04-15T21:38:45.911950+0000) 2024-04-15T21:38:52.901 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.71 MB/sec execute 80 sec latency 59831.405 ms 2024-04-15T21:38:53.901 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.66 MB/sec execute 81 sec latency 60831.460 ms 2024-04-15T21:38:54.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:53 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:53.720+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:54.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:53 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:53.720+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:54.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:53 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:53.720+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:14.113428+0000 front 2024-04-15T21:38:14.111353+0000 (oldest deadline 2024-04-15T21:38:40.011447+0000) 2024-04-15T21:38:54.114 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:53 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:53.720+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-15T21:38:20.011879+0000 front 2024-04-15T21:38:20.011835+0000 (oldest deadline 2024-04-15T21:38:45.911950+0000) 2024-04-15T21:38:54.901 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.62 MB/sec execute 82 sec latency 61831.483 ms 2024-04-15T21:38:55.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:54 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:54.692+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-15T21:37:58.909555+0000 front 2024-04-15T21:37:58.909536+0000 (oldest deadline 2024-04-15T21:38:24.809690+0000) 2024-04-15T21:38:55.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:54 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:54.692+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-15T21:38:04.809931+0000 front 2024-04-15T21:38:04.809951+0000 (oldest deadline 2024-04-15T21:38:30.710127+0000) 2024-04-15T21:38:55.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:54 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:54.692+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-15T21:38:14.113428+0000 front 2024-04-15T21:38:14.111353+0000 (oldest deadline 2024-04-15T21:38:40.011447+0000) 2024-04-15T21:38:55.113 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:54 smithi139 ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7[40232]: 2024-04-15T21:38:54.692+0000 7f64e33a7700 -1 osd.7 69 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-15T21:38:20.011879+0000 front 2024-04-15T21:38:20.011835+0000 (oldest deadline 2024-04-15T21:38:45.911950+0000) 2024-04-15T21:38:55.901 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.57 MB/sec execute 83 sec latency 62831.544 ms 2024-04-15T21:38:56.068 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:55 smithi139 podman[43745]: 2024-04-15 21:38:55.812358313 +0000 UTC m=+5.637783973 container died ae9b789871b1ba622a8e9d783aa1d40ea646502cf1f8dcdf4a8e71ad0c81937b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, version=8, distribution-scope=public, io.openshift.expose-services=, release=754, GIT_CLEAN=True) 2024-04-15T21:38:56.326 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:56 smithi139 podman[43745]: 2024-04-15 21:38:56.156545236 +0000 UTC m=+5.981970906 container cleanup ae9b789871b1ba622a8e9d783aa1d40ea646502cf1f8dcdf4a8e71ad0c81937b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, GIT_CLEAN=True, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-15T21:38:56.327 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:56 smithi139 bash[43745]: ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7 2024-04-15T21:38:56.595 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:56 smithi139 podman[43774]: 2024-04-15 21:38:56.324682936 +0000 UTC m=+0.510831434 container remove ae9b789871b1ba622a8e9d783aa1d40ea646502cf1f8dcdf4a8e71ad0c81937b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7, io.openshift.tags=base centos centos-stream, vcs-type=git, version=8, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, release=754, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, maintainer=Guillaume Abrioux , io.openshift.expose-services=, ceph=True, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8) 2024-04-15T21:38:56.595 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:56 smithi139 bash[43785]: Error: no container with name or ID "ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd.7" found: no such container 2024-04-15T21:38:56.863 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:56 smithi139 podman[43847]: 2024-04-15 21:38:56.496742655 +0000 UTC m=+0.018796910 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-15T21:38:56.864 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:56 smithi139 podman[43847]: 2024-04-15 21:38:56.600432511 +0000 UTC m=+0.122486780 container create 6ea8a39e22ba4e3d8687860562a29f18ac3d54f8b38e59da45ed5b8827764fbe (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, distribution-scope=public, io.buildah.version=1.19.8, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, version=8, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=) 2024-04-15T21:38:56.864 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:56 smithi139 podman[43847]: 2024-04-15 21:38:56.802925759 +0000 UTC m=+0.324979989 container init 6ea8a39e22ba4e3d8687860562a29f18ac3d54f8b38e59da45ed5b8827764fbe (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7-deactivate, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, RELEASE=HEAD, architecture=x86_64, ceph=True, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, name=centos-stream, release=754, io.openshift.tags=base centos centos-stream) 2024-04-15T21:38:56.864 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:56 smithi139 podman[43847]: 2024-04-15 21:38:56.806241091 +0000 UTC m=+0.328295319 container start 6ea8a39e22ba4e3d8687860562a29f18ac3d54f8b38e59da45ed5b8827764fbe (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, architecture=x86_64, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, distribution-scope=public, release=754, RELEASE=HEAD, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git) 2024-04-15T21:38:56.901 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.53 MB/sec execute 84 sec latency 63831.591 ms 2024-04-15T21:38:57.363 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:56 smithi139 podman[43847]: 2024-04-15 21:38:56.894049901 +0000 UTC m=+0.416104144 container attach 6ea8a39e22ba4e3d8687860562a29f18ac3d54f8b38e59da45ed5b8827764fbe (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7-deactivate, vcs-type=git, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.buildah.version=1.19.8, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, version=8, vendor=Red Hat, Inc., name=centos-stream, maintainer=Guillaume Abrioux , distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container) 2024-04-15T21:38:57.363 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 15 21:38:57 smithi139 podman[43847]: 2024-04-15 21:38:57.089051046 +0000 UTC m=+0.611105284 container died 6ea8a39e22ba4e3d8687860562a29f18ac3d54f8b38e59da45ed5b8827764fbe (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968-osd-7-deactivate, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, ceph=True, io.buildah.version=1.19.8, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, vendor=Red Hat, Inc., vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , distribution-scope=public, architecture=x86_64, name=centos-stream) 2024-04-15T21:38:57.675 DEBUG:teuthology.orchestra.run.smithi139:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e4a42038-fb6e-11ee-bc8f-c7b262605968@osd.7.service' 2024-04-15T21:38:57.717 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-15T21:38:57.717 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-15T21:38:57.718 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 --force --keep-logs 2024-04-15T21:38:57.902 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.49 MB/sec execute 85 sec latency 64831.617 ms 2024-04-15T21:38:58.902 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.45 MB/sec execute 86 sec latency 65831.942 ms 2024-04-15T21:38:59.902 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.41 MB/sec execute 87 sec latency 66831.965 ms 2024-04-15T21:39:00.902 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.37 MB/sec execute 88 sec latency 67832.267 ms 2024-04-15T21:39:01.902 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.33 MB/sec execute 89 sec latency 68832.294 ms 2024-04-15T21:39:02.902 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.30 MB/sec execute 90 sec latency 69832.319 ms 2024-04-15T21:39:03.902 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.26 MB/sec execute 91 sec latency 70832.385 ms 2024-04-15T21:39:04.902 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.22 MB/sec execute 92 sec latency 71832.414 ms 2024-04-15T21:39:05.902 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.19 MB/sec execute 93 sec latency 72832.469 ms 2024-04-15T21:39:06.902 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.16 MB/sec execute 94 sec latency 73832.517 ms 2024-04-15T21:39:07.902 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.12 MB/sec execute 95 sec latency 74832.561 ms 2024-04-15T21:39:08.902 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.09 MB/sec execute 96 sec latency 75832.597 ms 2024-04-15T21:39:09.902 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.06 MB/sec execute 97 sec latency 76832.648 ms 2024-04-15T21:39:10.902 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.03 MB/sec execute 98 sec latency 77832.686 ms 2024-04-15T21:39:11.903 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 3.00 MB/sec execute 99 sec latency 78832.726 ms 2024-04-15T21:39:12.903 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.97 MB/sec execute 100 sec latency 79832.767 ms 2024-04-15T21:39:13.903 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.94 MB/sec execute 101 sec latency 80832.806 ms 2024-04-15T21:39:14.903 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.91 MB/sec execute 102 sec latency 81832.858 ms 2024-04-15T21:39:15.903 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.88 MB/sec execute 103 sec latency 82832.896 ms 2024-04-15T21:39:16.903 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.85 MB/sec execute 104 sec latency 83832.950 ms 2024-04-15T21:39:17.903 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.83 MB/sec execute 105 sec latency 84833.009 ms 2024-04-15T21:39:18.903 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.80 MB/sec execute 106 sec latency 85833.050 ms 2024-04-15T21:39:19.903 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.77 MB/sec execute 107 sec latency 86833.099 ms 2024-04-15T21:39:20.903 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.75 MB/sec execute 108 sec latency 87833.146 ms 2024-04-15T21:39:21.903 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.72 MB/sec execute 109 sec latency 88833.186 ms 2024-04-15T21:39:22.903 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.70 MB/sec execute 110 sec latency 89833.225 ms 2024-04-15T21:39:23.903 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.67 MB/sec execute 111 sec latency 90833.243 ms 2024-04-15T21:39:24.903 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.65 MB/sec execute 112 sec latency 91833.291 ms 2024-04-15T21:39:25.903 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.63 MB/sec execute 113 sec latency 92833.307 ms 2024-04-15T21:39:26.903 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.60 MB/sec execute 114 sec latency 93833.325 ms 2024-04-15T21:39:27.359 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 --force --keep-logs 2024-04-15T21:39:27.904 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.58 MB/sec execute 115 sec latency 94833.430 ms 2024-04-15T21:39:28.903 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.56 MB/sec execute 116 sec latency 95833.470 ms 2024-04-15T21:39:29.904 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.54 MB/sec execute 117 sec latency 96833.568 ms 2024-04-15T21:39:30.904 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.51 MB/sec execute 118 sec latency 97833.679 ms 2024-04-15T21:39:31.904 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.49 MB/sec execute 119 sec latency 98833.765 ms 2024-04-15T21:39:32.904 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.47 MB/sec execute 120 sec latency 99833.865 ms 2024-04-15T21:39:33.904 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.45 MB/sec execute 121 sec latency 100833.983 ms 2024-04-15T21:39:34.904 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.43 MB/sec execute 122 sec latency 101834.067 ms 2024-04-15T21:39:35.904 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.41 MB/sec execute 123 sec latency 102834.178 ms 2024-04-15T21:39:36.904 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.39 MB/sec execute 124 sec latency 103834.268 ms 2024-04-15T21:39:37.904 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.37 MB/sec execute 125 sec latency 104834.345 ms 2024-04-15T21:39:38.904 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.35 MB/sec execute 126 sec latency 105834.421 ms 2024-04-15T21:39:39.904 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.34 MB/sec execute 127 sec latency 106834.486 ms 2024-04-15T21:39:40.904 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.32 MB/sec execute 128 sec latency 107834.539 ms 2024-04-15T21:39:41.905 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.30 MB/sec execute 129 sec latency 108834.602 ms 2024-04-15T21:39:42.905 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.28 MB/sec execute 130 sec latency 109834.713 ms 2024-04-15T21:39:43.905 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.26 MB/sec execute 131 sec latency 110834.824 ms 2024-04-15T21:39:44.905 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.25 MB/sec execute 132 sec latency 111834.897 ms 2024-04-15T21:39:45.905 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.23 MB/sec execute 133 sec latency 112834.973 ms 2024-04-15T21:39:46.905 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.21 MB/sec execute 134 sec latency 113835.080 ms 2024-04-15T21:39:47.531 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-15T21:39:47.568 DEBUG:teuthology.orchestra.run.smithi139:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-15T21:39:47.599 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-15T21:39:47.601 DEBUG:teuthology.misc:Transferring archived files from smithi083:/var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-15_21:08:03-orch-squid-distro-default-smithi/7657448/remote/smithi083/crash 2024-04-15T21:39:47.602 DEBUG:teuthology.orchestra.run.smithi083:> sudo tar c -f - -C /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/crash -- . 2024-04-15T21:39:47.655 INFO:teuthology.orchestra.run.smithi083.stderr:tar: /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/crash: Cannot open: No such file or directory 2024-04-15T21:39:47.656 INFO:teuthology.orchestra.run.smithi083.stderr:tar: Error is not recoverable: exiting now 2024-04-15T21:39:47.658 DEBUG:teuthology.misc:Transferring archived files from smithi139:/var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-15_21:08:03-orch-squid-distro-default-smithi/7657448/remote/smithi139/crash 2024-04-15T21:39:47.659 DEBUG:teuthology.orchestra.run.smithi139:> sudo tar c -f - -C /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/crash -- . 2024-04-15T21:39:47.699 INFO:teuthology.orchestra.run.smithi139.stderr:tar: /var/lib/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/crash: Cannot open: No such file or directory 2024-04-15T21:39:47.700 INFO:teuthology.orchestra.run.smithi139.stderr:tar: Error is not recoverable: exiting now 2024-04-15T21:39:47.701 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-15T21:39:47.701 DEBUG:teuthology.orchestra.run.smithi083:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-15T21:39:47.733 INFO:teuthology.orchestra.run.smithi083.stdout:2024-04-15T21:35:56.886958+0000 mon.smithi083 (mon.0) 749 : cluster [WRN] Health check failed: Upgrade: failed to pull target image (UPGRADE_FAILED_PULL) 2024-04-15T21:39:47.734 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-15T21:39:47.734 INFO:tasks.cephadm:Compressing logs... 2024-04-15T21:39:47.734 DEBUG:teuthology.orchestra.run.smithi083:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-15T21:39:47.776 DEBUG:teuthology.orchestra.run.smithi139:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-15T21:39:47.802 INFO:teuthology.orchestra.run.smithi083.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-15T21:39:47.803 INFO:teuthology.orchestra.run.smithi083.stderr:: No such file or directory 2024-04-15T21:39:47.804 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-mon.smithi083.log 2024-04-15T21:39:47.804 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph.audit.log 2024-04-15T21:39:47.805 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/cephadm.log: /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-mon.smithi083.log: gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.1.log 2024-04-15T21:39:47.805 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-mds.foofs.smithi083.lgyynu.log 2024-04-15T21:39:47.805 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph.audit.log: /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.3.log 2024-04-15T21:39:47.805 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.0.log 2024-04-15T21:39:47.812 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-mds.foofs.smithi083.lgyynu.log: /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.3.log: 81.7% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-mds.foofs.smithi083.lgyynu.log.gz 2024-04-15T21:39:47.812 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph.cephadm.log 2024-04-15T21:39:47.812 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph.log 2024-04-15T21:39:47.812 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.2.log 2024-04-15T21:39:47.812 INFO:teuthology.orchestra.run.smithi083.stderr: 83.0% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph.cephadm.log.gz 2024-04-15T21:39:47.812 INFO:teuthology.orchestra.run.smithi083.stderr: 90.8%/var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph.log: -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph.audit.log.gz 2024-04-15T21:39:47.813 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-volume.log 2024-04-15T21:39:47.813 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-mgr.smithi083.kxqarn.log 2024-04-15T21:39:47.813 INFO:teuthology.orchestra.run.smithi083.stderr: 86.8%/var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-volume.log: -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph.log.gz 2024-04-15T21:39:47.815 INFO:teuthology.orchestra.run.smithi139.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-15T21:39:47.815 INFO:teuthology.orchestra.run.smithi139.stderr:: No such file or directory 2024-04-15T21:39:47.815 INFO:teuthology.orchestra.run.smithi139.stderr:gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.6.log 2024-04-15T21:39:47.815 INFO:teuthology.orchestra.run.smithi139.stderr:gzip -5/var/log/ceph/cephadm.log: --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-mds.foofs.smithi139.akqukn.log 2024-04-15T21:39:47.815 INFO:teuthology.orchestra.run.smithi139.stderr:/var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph.audit.log 2024-04-15T21:39:47.815 INFO:teuthology.orchestra.run.smithi139.stderr:/var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-mds.foofs.smithi139.akqukn.log: gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-mon.smithi139.log 2024-04-15T21:39:47.815 INFO:teuthology.orchestra.run.smithi139.stderr: 60.6% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-mds.foofs.smithi139.akqukn.log.gz 2024-04-15T21:39:47.816 INFO:teuthology.orchestra.run.smithi139.stderr:gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.4.log 2024-04-15T21:39:47.816 INFO:teuthology.orchestra.run.smithi139.stderr:/var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph.audit.log: /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-mon.smithi139.log: gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.7.log 2024-04-15T21:39:47.816 INFO:teuthology.orchestra.run.smithi139.stderr:/var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.5.log 2024-04-15T21:39:47.816 INFO:teuthology.orchestra.run.smithi139.stderr:/var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-mgr.smithi139.bqqykn.log 2024-04-15T21:39:47.816 INFO:teuthology.orchestra.run.smithi139.stderr:/var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.5.log: 91.0% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph.audit.log.gzgzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph.cephadm.log 2024-04-15T21:39:47.816 INFO:teuthology.orchestra.run.smithi139.stderr: 2024-04-15T21:39:47.816 INFO:teuthology.orchestra.run.smithi139.stderr:/var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-mgr.smithi139.bqqykn.log: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-15T21:39:47.816 INFO:teuthology.orchestra.run.smithi139.stderr: 92.4% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-mgr.smithi139.bqqykn.log.gz 2024-04-15T21:39:47.817 INFO:teuthology.orchestra.run.smithi139.stderr:gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph.log 2024-04-15T21:39:47.817 INFO:teuthology.orchestra.run.smithi139.stderr:/var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-volume.log 2024-04-15T21:39:47.817 INFO:teuthology.orchestra.run.smithi139.stderr: 81.9% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph.cephadm.log.gz 2024-04-15T21:39:47.817 INFO:teuthology.orchestra.run.smithi139.stderr:/var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph.log: /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-volume.log: 86.8% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph.log.gz 2024-04-15T21:39:47.818 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-mgr.smithi083.kxqarn.log: 91.1% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-volume.log.gz 2024-04-15T21:39:47.818 INFO:teuthology.orchestra.run.smithi083.stderr: 91.7% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-15T21:39:47.820 INFO:teuthology.orchestra.run.smithi139.stderr: 90.9% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-volume.log.gz 2024-04-15T21:39:47.905 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.20 MB/sec execute 135 sec latency 114835.123 ms 2024-04-15T21:39:47.951 INFO:teuthology.orchestra.run.smithi139.stderr: 92.7% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-mon.smithi139.log.gz 2024-04-15T21:39:47.965 INFO:teuthology.orchestra.run.smithi083.stderr: 88.8% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-mgr.smithi083.kxqarn.log.gz 2024-04-15T21:39:48.441 INFO:teuthology.orchestra.run.smithi083.stderr: 91.9% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-mon.smithi083.log.gz 2024-04-15T21:39:48.905 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.18 MB/sec execute 136 sec latency 115835.166 ms 2024-04-15T21:39:49.905 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.17 MB/sec execute 137 sec latency 116835.216 ms 2024-04-15T21:39:50.029 INFO:teuthology.orchestra.run.smithi083.stderr: 94.6% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.2.log.gz 2024-04-15T21:39:50.905 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.15 MB/sec execute 138 sec latency 117835.277 ms 2024-04-15T21:39:51.657 INFO:teuthology.orchestra.run.smithi139.stderr: 94.3% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.6.log.gz 2024-04-15T21:39:51.886 INFO:teuthology.orchestra.run.smithi139.stderr: 94.0% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.4.log.gz 2024-04-15T21:39:51.905 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.13 MB/sec execute 139 sec latency 118835.308 ms 2024-04-15T21:39:52.195 INFO:teuthology.orchestra.run.smithi083.stderr: 94.3% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.1.log.gz 2024-04-15T21:39:52.223 INFO:teuthology.orchestra.run.smithi083.stderr: 94.1% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.3.log.gz 2024-04-15T21:39:52.755 INFO:teuthology.orchestra.run.smithi083.stderr: 94.2% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.0.log.gz 2024-04-15T21:39:52.757 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-15T21:39:52.757 INFO:teuthology.orchestra.run.smithi083.stderr:real 0m4.968s 2024-04-15T21:39:52.757 INFO:teuthology.orchestra.run.smithi083.stderr:user 0m16.160s 2024-04-15T21:39:52.757 INFO:teuthology.orchestra.run.smithi083.stderr:sys 0m0.643s 2024-04-15T21:39:52.778 INFO:teuthology.orchestra.run.smithi139.stderr: 94.3% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.5.log.gz 2024-04-15T21:39:52.905 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.12 MB/sec execute 140 sec latency 119835.386 ms 2024-04-15T21:39:53.906 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.10 MB/sec execute 141 sec latency 120835.551 ms 2024-04-15T21:39:54.906 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.09 MB/sec execute 142 sec latency 121835.644 ms 2024-04-15T21:39:55.238 INFO:teuthology.orchestra.run.smithi139.stderr: 94.2% -- replaced with /var/log/ceph/e4a42038-fb6e-11ee-bc8f-c7b262605968/ceph-osd.7.log.gz 2024-04-15T21:39:55.240 INFO:teuthology.orchestra.run.smithi139.stderr: 2024-04-15T21:39:55.240 INFO:teuthology.orchestra.run.smithi139.stderr:real 0m7.449s 2024-04-15T21:39:55.240 INFO:teuthology.orchestra.run.smithi139.stderr:user 0m19.720s 2024-04-15T21:39:55.241 INFO:teuthology.orchestra.run.smithi139.stderr:sys 0m0.777s 2024-04-15T21:39:55.241 INFO:tasks.cephadm:Archiving logs... 2024-04-15T21:39:55.241 DEBUG:teuthology.misc:Transferring archived files from smithi083:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-15_21:08:03-orch-squid-distro-default-smithi/7657448/remote/smithi083/log 2024-04-15T21:39:55.242 DEBUG:teuthology.orchestra.run.smithi083:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-15T21:39:55.927 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.07 MB/sec execute 143 sec latency 122835.760 ms 2024-04-15T21:39:56.934 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.06 MB/sec execute 144 sec latency 123835.853 ms 2024-04-15T21:39:57.597 DEBUG:teuthology.misc:Transferring archived files from smithi139:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-15_21:08:03-orch-squid-distro-default-smithi/7657448/remote/smithi139/log 2024-04-15T21:39:57.598 DEBUG:teuthology.orchestra.run.smithi139:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-15T21:39:57.910 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.05 MB/sec execute 145 sec latency 124835.912 ms 2024-04-15T21:39:58.910 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.03 MB/sec execute 146 sec latency 125836.027 ms 2024-04-15T21:39:59.910 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.02 MB/sec execute 147 sec latency 126836.136 ms 2024-04-15T21:40:00.490 INFO:tasks.cephadm:Removing cluster... 2024-04-15T21:40:00.490 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 --force 2024-04-15T21:40:00.800 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 --force 2024-04-15T21:40:00.906 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 2.00 MB/sec execute 148 sec latency 127836.254 ms 2024-04-15T21:40:01.102 INFO:tasks.cephadm:Removing cephadm ... 2024-04-15T21:40:01.103 DEBUG:teuthology.orchestra.run.smithi083:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-15T21:40:01.119 DEBUG:teuthology.orchestra.run.smithi139:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-15T21:40:01.134 INFO:tasks.cephadm:Teardown complete 2024-04-15T21:40:01.135 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-15T21:40:01.146 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-15T21:40:01.147 DEBUG:teuthology.orchestra.run.smithi083:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-15T21:40:01.162 DEBUG:teuthology.orchestra.run.smithi139:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-15T21:40:01.174 INFO:teuthology.orchestra.run.smithi083.stderr:bash: line 1: ntpq: command not found 2024-04-15T21:40:01.181 INFO:teuthology.orchestra.run.smithi083.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-15T21:40:01.181 INFO:teuthology.orchestra.run.smithi083.stdout:=============================================================================== 2024-04-15T21:40:01.182 INFO:teuthology.orchestra.run.smithi083.stdout:^* hv01.front.sepia.ceph.com 3 6 377 62 +777us[ +775us] +/- 46ms 2024-04-15T21:40:01.182 INFO:teuthology.orchestra.run.smithi083.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 62 -314us[ -314us] +/- 58ms 2024-04-15T21:40:01.183 INFO:teuthology.orchestra.run.smithi083.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 62 -928us[ -928us] +/- 71ms 2024-04-15T21:40:01.183 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-04-15T21:40:01.189 INFO:teuthology.orchestra.run.smithi139.stderr:bash: line 1: ntpq: command not found 2024-04-15T21:40:01.194 INFO:teuthology.orchestra.run.smithi139.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-15T21:40:01.194 INFO:teuthology.orchestra.run.smithi139.stdout:=============================================================================== 2024-04-15T21:40:01.194 INFO:teuthology.orchestra.run.smithi139.stdout:^* hv01.front.sepia.ceph.com 3 6 377 63 +807us[ +798us] +/- 46ms 2024-04-15T21:40:01.194 INFO:teuthology.orchestra.run.smithi139.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 62 -277us[ -277us] +/- 58ms 2024-04-15T21:40:01.194 INFO:teuthology.orchestra.run.smithi139.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 63 -900us[ -908us] +/- 71ms 2024-04-15T21:40:01.194 INFO:teuthology.orchestra.run.smithi139.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-04-15T21:40:01.195 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-15T21:40:01.204 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-15T21:40:01.206 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-15T21:40:01.229 DEBUG:teuthology.orchestra.run.smithi083:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-15T21:40:01.304 DEBUG:teuthology.orchestra.run.smithi139:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-15T21:40:01.376 DEBUG:teuthology.orchestra.run.smithi083:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-15T21:40:01.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-15T21:40:01.408 DEBUG:teuthology.orchestra.run.smithi139:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-15T21:40:01.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-15T21:40:01.439 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-15T21:40:01.450 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-15T21:40:01.490 INFO:teuthology.task.internal:Duration was 1043.880027 seconds 2024-04-15T21:40:01.491 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-15T21:40:01.517 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-15T21:40:01.517 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-15T21:40:01.520 DEBUG:teuthology.orchestra.run.smithi139:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-15T21:40:01.563 INFO:teuthology.orchestra.run.smithi083.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-15T21:40:01.565 INFO:teuthology.orchestra.run.smithi139.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-15T21:40:01.906 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 1.99 MB/sec execute 149 sec latency 128836.299 ms 2024-04-15T21:40:01.907 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-15T21:40:01.907 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi083.front.sepia.ceph.com 2024-04-15T21:40:01.907 DEBUG:teuthology.orchestra.run.smithi083:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-15T21:40:01.929 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi139.front.sepia.ceph.com 2024-04-15T21:40:01.930 DEBUG:teuthology.orchestra.run.smithi139:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-15T21:40:01.958 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-15T21:40:01.959 DEBUG:teuthology.orchestra.run.smithi083:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-15T21:40:01.973 DEBUG:teuthology.orchestra.run.smithi139:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-15T21:40:02.028 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-15T21:40:02.028 DEBUG:teuthology.orchestra.run.smithi083:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-15T21:40:02.044 DEBUG:teuthology.orchestra.run.smithi139:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-15T21:40:02.906 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 1.98 MB/sec execute 150 sec latency 129836.359 ms 2024-04-15T21:40:03.906 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 1.96 MB/sec execute 151 sec latency 130836.424 ms 2024-04-15T21:40:04.906 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 1.95 MB/sec execute 152 sec latency 131836.485 ms 2024-04-15T21:40:05.906 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 1.94 MB/sec execute 153 sec latency 132836.551 ms 2024-04-15T21:40:06.783 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-15T21:40:06.793 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-15T21:40:06.793 DEBUG:teuthology.orchestra.run.smithi083:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-15T21:40:06.821 DEBUG:teuthology.orchestra.run.smithi139:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-15T21:40:06.855 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-15T21:40:06.865 DEBUG:teuthology.orchestra.run.smithi083:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-15T21:40:06.868 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-15T21:40:06.892 INFO:teuthology.orchestra.run.smithi083.stdout:kernel.core_pattern = core 2024-04-15T21:40:06.906 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 1.93 MB/sec execute 154 sec latency 133836.578 ms 2024-04-15T21:40:06.927 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core 2024-04-15T21:40:06.959 DEBUG:teuthology.orchestra.run.smithi083:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-15T21:40:06.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-15T21:40:06.992 DEBUG:teuthology.orchestra.run.smithi139:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-15T21:40:07.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-15T21:40:07.015 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-15T21:40:07.028 INFO:teuthology.task.internal:Transferring archived files... 2024-04-15T21:40:07.029 DEBUG:teuthology.misc:Transferring archived files from smithi083:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-15_21:08:03-orch-squid-distro-default-smithi/7657448/remote/smithi083 2024-04-15T21:40:07.030 DEBUG:teuthology.orchestra.run.smithi083:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-15T21:40:07.088 DEBUG:teuthology.misc:Transferring archived files from smithi139:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-15_21:08:03-orch-squid-distro-default-smithi/7657448/remote/smithi139 2024-04-15T21:40:07.088 DEBUG:teuthology.orchestra.run.smithi139:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-15T21:40:07.134 INFO:teuthology.task.internal:Removing archive directory... 2024-04-15T21:40:07.134 DEBUG:teuthology.orchestra.run.smithi083:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-15T21:40:07.137 DEBUG:teuthology.orchestra.run.smithi139:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-15T21:40:07.187 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-15T21:40:07.198 INFO:teuthology.task.internal:Not uploading archives. 2024-04-15T21:40:07.198 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-15T21:40:07.209 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-15T21:40:07.209 DEBUG:teuthology.orchestra.run.smithi083:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-15T21:40:07.212 DEBUG:teuthology.orchestra.run.smithi139:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-15T21:40:07.225 INFO:teuthology.orchestra.run.smithi083.stdout: 265305 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 15 21:40 /home/ubuntu/cephtest 2024-04-15T21:40:07.243 INFO:teuthology.orchestra.run.smithi139.stdout: 265305 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 15 21:40 /home/ubuntu/cephtest 2024-04-15T21:40:07.244 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-15T21:40:07.255 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-15T21:40:07.371 INFO:teuthology.nuke:Checking targets against current locks 2024-04-15T21:40:07.399 DEBUG:teuthology.nuke:shortname: smithi083 2024-04-15T21:40:07.399 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-15T21:40:07.424 DEBUG:teuthology.nuke:shortname: smithi139 2024-04-15T21:40:07.424 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-15T21:40:07.428 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi083.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-15_21:08:03-orch-squid-distro-default-smithi/7657448', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-15 21:14:35.079515', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNUbAozp6fZ0b5gIPjYHw29AUj/hOYr3KTuGHNYVL0yOOb3LB5Z73TwEcdvFlDV6p9VadW2GtPIhl8yTfj1gvdQ='} 2024-04-15T21:40:07.452 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi139.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-15_21:08:03-orch-squid-distro-default-smithi/7657448', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-15 21:14:35.081276', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMQn2Rjol8cML9HuaQ7TgtO3uAw2/xk0vArkIKOQNNOm/E7SdbgQkyl8W+3uIU/5V98z1ar54u8GyjH0fx3J120='} 2024-04-15T21:40:07.479 INFO:teuthology.orchestra.console.smithi083:Power off 2024-04-15T21:40:07.479 DEBUG:teuthology.orchestra.console.smithi083:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-15T21:40:07.508 DEBUG:teuthology.orchestra.console.smithi083:power off output: Chassis Power Control: Down/Off 2024-04-15T21:40:07.509 DEBUG:teuthology.orchestra.console.smithi083:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-15T21:40:07.543 INFO:teuthology.orchestra.console.smithi139:Power off 2024-04-15T21:40:07.543 DEBUG:teuthology.orchestra.console.smithi139:pexpect command: ipmitool -H smithi139.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-15T21:40:07.556 DEBUG:teuthology.orchestra.console.smithi083:check power output: Chassis Power is on 2024-04-15T21:40:07.568 DEBUG:teuthology.orchestra.console.smithi139:power off output: Chassis Power Control: Down/Off 2024-04-15T21:40:07.568 DEBUG:teuthology.orchestra.console.smithi139:pexpect command: ipmitool -H smithi139.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-15T21:40:07.594 DEBUG:teuthology.orchestra.console.smithi139:check power output: Chassis Power is on 2024-04-15T21:40:07.907 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 1.91 MB/sec execute 155 sec latency 134836.645 ms 2024-04-15T21:40:08.907 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 1.90 MB/sec execute 156 sec latency 135836.692 ms 2024-04-15T21:40:09.907 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81965 1.89 MB/sec execute 157 sec latency 136836.771 ms 2024-04-15T21:40:10.707 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-15T21:40:11.557 DEBUG:teuthology.orchestra.console.smithi083:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-15T21:40:11.595 DEBUG:teuthology.orchestra.console.smithi139:pexpect command: ipmitool -H smithi139.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-15T21:40:11.669 DEBUG:teuthology.orchestra.console.smithi083:check power output: Chassis Power is on 2024-04-15T21:40:11.706 DEBUG:teuthology.orchestra.console.smithi139:check power output: Chassis Power is on 2024-04-15T21:40:15.674 DEBUG:teuthology.orchestra.console.smithi083:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-15T21:40:15.707 DEBUG:teuthology.orchestra.console.smithi139:pexpect command: ipmitool -H smithi139.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-15T21:40:15.785 DEBUG:teuthology.orchestra.console.smithi083:check power output: Chassis Power is off 2024-04-15T21:40:15.819 DEBUG:teuthology.orchestra.console.smithi139:check power output: Chassis Power is off 2024-04-15T21:40:15.887 INFO:teuthology.orchestra.console.smithi083:Power off completed 2024-04-15T21:40:15.921 INFO:teuthology.orchestra.console.smithi139:Power off completed 2024-04-15T21:40:16.027 INFO:teuthology.run:Summary data: description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1043.8800265789032 failure_reason: 'Command failed on smithi083 with status 1: ''sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e4a42038-fb6e-11ee-bc8f-c7b262605968 -e sha1=a9a752df26c63acad72e1b3569fd79a515ca0765 -- bash -c \''ceph versions | jq -e \''"\''"\''.overall | keys\''"\''"\'' | grep $sha1\''''' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=fcf64b3f13ce49289de702f7351883a1 status: fail success: false 2024-04-15T21:40:16.027 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-15T21:40:17.126 INFO:teuthology.run:FAIL