2024-05-08T06:18:48.604 INFO:root:teuthology version: 0.0.1.dev271+g1ae7ad8 2024-05-08T06:18:48.604 DEBUG:teuthology.run:Teuthology command: teuthology --name adking-2024-05-08_05:24:58-orch:cephadm-wip-adk-testing-2024-05-07-0802-distro-default-smithi --description orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} --verbose --archive /home/teuthworker/archive/adking-2024-05-08_05:24:58-orch:cephadm-wip-adk-testing-2024-05-07-0802-distro-default-smithi/7697317 --owner scheduled_adking@teuthology -- /home/teuthworker/archive/adking-2024-05-08_05:24:58-orch:cephadm-wip-adk-testing-2024-05-07-0802-distro-default-smithi/7697317/orig.config.yaml 2024-05-08T06:18:48.643 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-05-08T06:18:48.732 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/adking-2024-05-08_05:24:58-orch:cephadm-wip-adk-testing-2024-05-07-0802-distro-default-smithi/7697317 branch: wip-adk-testing-2024-05-07-0802 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: adking@redhat.com first_in_suite: false job_id: '7697317' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: adking-2024-05-08_05:24:58-orch:cephadm-wip-adk-testing-2024-05-07-0802-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-adk-testing-2024-05-07-0802 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: 7c8f650b36e258f639fa4a83becade57cbfd2009 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 7c8f650b36e258f639fa4a83becade57cbfd2009 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: wip-adk-testing-2024-05-07-0802 sha1: 7c8f650b36e258f639fa4a83becade57cbfd2009 owner: scheduled_adking@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 1064 sha1: 7c8f650b36e258f639fa4a83becade57cbfd2009 sleep_before_teardown: 0 subset: 1/20 suite: orch:cephadm suite_branch: wip-adk-testing-2024-05-07-0802 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_7c8f650b36e258f639fa4a83becade57cbfd2009/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 7c8f650b36e258f639fa4a83becade57cbfd2009 targets: smithi082.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGY1qVRgCmeZi86MEu9yJJhTrNkjNOU1rLfL+l8aLBNLXBXdIFVxBJkgsRCDREIVpXQU1QE2qkNu7HyIzfj7E1M= smithi134.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBChUJ8uXPx39LBsFglaZqzd30QXjfWAQm3+k2zF+FcKuUVlBC6Shopwj017QS7+eyg8ge8/tOnMOdKN+pOEjXBs= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 1ae7ad82388e92a475afff437d49054826c019a1 timestamp: 2024-05-08_05:24:58 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: adking verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-05-08T06:18:48.732 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_7c8f650b36e258f639fa4a83becade57cbfd2009/qa; will attempt to use it 2024-05-08T06:18:48.732 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_7c8f650b36e258f639fa4a83becade57cbfd2009/qa/tasks 2024-05-08T06:18:48.733 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-05-08T06:18:48.734 INFO:teuthology.task.internal:Checking packages... 2024-05-08T06:18:48.759 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '7c8f650b36e258f639fa4a83becade57cbfd2009' 2024-05-08T06:18:48.759 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-05-08T06:18:48.759 INFO:teuthology.packaging:ref: None 2024-05-08T06:18:48.759 INFO:teuthology.packaging:tag: None 2024-05-08T06:18:48.759 INFO:teuthology.packaging:branch: wip-adk-testing-2024-05-07-0802 2024-05-08T06:18:48.759 INFO:teuthology.packaging:sha1: 7c8f650b36e258f639fa4a83becade57cbfd2009 2024-05-08T06:18:48.759 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-adk-testing-2024-05-07-0802 2024-05-08T06:18:48.982 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-3593.g7c8f650b 2024-05-08T06:18:48.984 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-05-08T06:18:48.993 INFO:teuthology.task.internal:no buildpackages task found 2024-05-08T06:18:48.993 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-05-08T06:18:49.029 INFO:teuthology.task.internal:Saving configuration 2024-05-08T06:18:49.045 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-05-08T06:18:49.054 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-05-08T06:18:49.081 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi082.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-05-08_05:24:58-orch:cephadm-wip-adk-testing-2024-05-07-0802-distro-default-smithi/7697317', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-08 06:13:39.939450', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGY1qVRgCmeZi86MEu9yJJhTrNkjNOU1rLfL+l8aLBNLXBXdIFVxBJkgsRCDREIVpXQU1QE2qkNu7HyIzfj7E1M='} 2024-05-08T06:18:49.106 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi134.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-05-08_05:24:58-orch:cephadm-wip-adk-testing-2024-05-07-0802-distro-default-smithi/7697317', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-08 06:13:39.940982', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBChUJ8uXPx39LBsFglaZqzd30QXjfWAQm3+k2zF+FcKuUVlBC6Shopwj017QS7+eyg8ge8/tOnMOdKN+pOEjXBs='} 2024-05-08T06:18:49.106 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-05-08T06:18:49.115 INFO:teuthology.task.internal:roles: ubuntu@smithi082.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-05-08T06:18:49.115 INFO:teuthology.task.internal:roles: ubuntu@smithi134.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-05-08T06:18:49.115 INFO:teuthology.run_tasks:Running task console_log... 2024-05-08T06:18:49.209 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f913889d160>, signals=[15]) 2024-05-08T06:18:49.210 INFO:teuthology.run_tasks:Running task internal.connect... 2024-05-08T06:18:49.219 INFO:teuthology.task.internal:Opening connections... 2024-05-08T06:18:49.220 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi082.front.sepia.ceph.com 2024-05-08T06:18:49.222 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi082.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-08T06:18:49.336 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi134.front.sepia.ceph.com 2024-05-08T06:18:49.337 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-08T06:18:49.414 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-05-08T06:18:49.421 DEBUG:teuthology.orchestra.run.smithi082:> uname -m 2024-05-08T06:18:49.439 INFO:teuthology.orchestra.run.smithi082.stdout:x86_64 2024-05-08T06:18:49.439 DEBUG:teuthology.orchestra.run.smithi082:> cat /etc/os-release 2024-05-08T06:18:49.496 INFO:teuthology.orchestra.run.smithi082.stdout:NAME="CentOS Stream" 2024-05-08T06:18:49.496 INFO:teuthology.orchestra.run.smithi082.stdout:VERSION="9" 2024-05-08T06:18:49.496 INFO:teuthology.orchestra.run.smithi082.stdout:ID="centos" 2024-05-08T06:18:49.496 INFO:teuthology.orchestra.run.smithi082.stdout:ID_LIKE="rhel fedora" 2024-05-08T06:18:49.496 INFO:teuthology.orchestra.run.smithi082.stdout:VERSION_ID="9" 2024-05-08T06:18:49.496 INFO:teuthology.orchestra.run.smithi082.stdout:PLATFORM_ID="platform:el9" 2024-05-08T06:18:49.496 INFO:teuthology.orchestra.run.smithi082.stdout:PRETTY_NAME="CentOS Stream 9" 2024-05-08T06:18:49.496 INFO:teuthology.orchestra.run.smithi082.stdout:ANSI_COLOR="0;31" 2024-05-08T06:18:49.496 INFO:teuthology.orchestra.run.smithi082.stdout:LOGO="fedora-logo-icon" 2024-05-08T06:18:49.496 INFO:teuthology.orchestra.run.smithi082.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-05-08T06:18:49.497 INFO:teuthology.orchestra.run.smithi082.stdout:HOME_URL="https://centos.org/" 2024-05-08T06:18:49.497 INFO:teuthology.orchestra.run.smithi082.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-05-08T06:18:49.497 INFO:teuthology.orchestra.run.smithi082.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-05-08T06:18:49.497 INFO:teuthology.orchestra.run.smithi082.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-05-08T06:18:49.497 INFO:teuthology.lock.ops:Updating smithi082.front.sepia.ceph.com on lock server 2024-05-08T06:18:49.516 DEBUG:teuthology.orchestra.run.smithi134:> uname -m 2024-05-08T06:18:49.534 INFO:teuthology.orchestra.run.smithi134.stdout:x86_64 2024-05-08T06:18:49.534 DEBUG:teuthology.orchestra.run.smithi134:> cat /etc/os-release 2024-05-08T06:18:49.591 INFO:teuthology.orchestra.run.smithi134.stdout:NAME="CentOS Stream" 2024-05-08T06:18:49.591 INFO:teuthology.orchestra.run.smithi134.stdout:VERSION="9" 2024-05-08T06:18:49.591 INFO:teuthology.orchestra.run.smithi134.stdout:ID="centos" 2024-05-08T06:18:49.591 INFO:teuthology.orchestra.run.smithi134.stdout:ID_LIKE="rhel fedora" 2024-05-08T06:18:49.591 INFO:teuthology.orchestra.run.smithi134.stdout:VERSION_ID="9" 2024-05-08T06:18:49.591 INFO:teuthology.orchestra.run.smithi134.stdout:PLATFORM_ID="platform:el9" 2024-05-08T06:18:49.591 INFO:teuthology.orchestra.run.smithi134.stdout:PRETTY_NAME="CentOS Stream 9" 2024-05-08T06:18:49.591 INFO:teuthology.orchestra.run.smithi134.stdout:ANSI_COLOR="0;31" 2024-05-08T06:18:49.591 INFO:teuthology.orchestra.run.smithi134.stdout:LOGO="fedora-logo-icon" 2024-05-08T06:18:49.591 INFO:teuthology.orchestra.run.smithi134.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-05-08T06:18:49.592 INFO:teuthology.orchestra.run.smithi134.stdout:HOME_URL="https://centos.org/" 2024-05-08T06:18:49.592 INFO:teuthology.orchestra.run.smithi134.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-05-08T06:18:49.592 INFO:teuthology.orchestra.run.smithi134.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-05-08T06:18:49.592 INFO:teuthology.orchestra.run.smithi134.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-05-08T06:18:49.592 INFO:teuthology.lock.ops:Updating smithi134.front.sepia.ceph.com on lock server 2024-05-08T06:18:49.607 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-05-08T06:18:49.619 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-05-08T06:18:49.633 INFO:teuthology.task.internal:Checking for old test directory... 2024-05-08T06:18:49.633 DEBUG:teuthology.orchestra.run.smithi082:> test '!' -e /home/ubuntu/cephtest 2024-05-08T06:18:49.636 DEBUG:teuthology.orchestra.run.smithi134:> test '!' -e /home/ubuntu/cephtest 2024-05-08T06:18:49.651 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-05-08T06:18:49.662 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-05-08T06:18:49.662 DEBUG:teuthology.orchestra.run.smithi082:> test -z $(ls -A /var/lib/ceph) 2024-05-08T06:18:49.692 DEBUG:teuthology.orchestra.run.smithi134:> test -z $(ls -A /var/lib/ceph) 2024-05-08T06:18:49.776 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-05-08T06:18:49.829 INFO:teuthology.run_tasks:Running task kernel... 2024-05-08T06:18:49.845 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-05-08T06:18:49.845 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-05-08T06:18:49.846 DEBUG:teuthology.orchestra.run.smithi082:> test -f /run/.containerenv -o -f /.dockerenv 2024-05-08T06:18:49.846 DEBUG:teuthology.orchestra.run.smithi134:> test -f /run/.containerenv -o -f /.dockerenv 2024-05-08T06:18:49.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-08T06:18:49.862 DEBUG:teuthology.orchestra.run.smithi134:> uname -r 2024-05-08T06:18:49.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-08T06:18:49.863 DEBUG:teuthology.orchestra.run.smithi082:> uname -r 2024-05-08T06:18:49.918 INFO:teuthology.orchestra.run.smithi134.stdout:5.14.0-437.el9.x86_64 2024-05-08T06:18:49.918 INFO:teuthology.task.kernel:Running kernel on smithi134: 5.14.0-437.el9.x86_64 2024-05-08T06:18:49.918 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum install -y kernel 2024-05-08T06:18:49.920 INFO:teuthology.orchestra.run.smithi082.stdout:5.14.0-437.el9.x86_64 2024-05-08T06:18:49.920 INFO:teuthology.task.kernel:Running kernel on smithi082: 5.14.0-437.el9.x86_64 2024-05-08T06:18:49.920 DEBUG:teuthology.orchestra.run.smithi082:> sudo yum install -y kernel 2024-05-08T06:18:52.388 INFO:teuthology.orchestra.run.smithi082.stdout:CentOS Stream 9 - BaseOS 10 MB/s | 8.1 MB 00:00 2024-05-08T06:18:52.476 INFO:teuthology.orchestra.run.smithi134.stdout:CentOS Stream 9 - BaseOS 9.1 MB/s | 8.1 MB 00:00 2024-05-08T06:18:55.663 INFO:teuthology.orchestra.run.smithi082.stdout:CentOS Stream 9 - AppStream 8.8 MB/s | 19 MB 00:02 2024-05-08T06:18:55.676 INFO:teuthology.orchestra.run.smithi134.stdout:CentOS Stream 9 - AppStream 9.1 MB/s | 19 MB 00:02 2024-05-08T06:19:00.080 INFO:teuthology.orchestra.run.smithi082.stdout:CentOS Stream 9 - CRB 7.4 MB/s | 6.2 MB 00:00 2024-05-08T06:19:00.143 INFO:teuthology.orchestra.run.smithi134.stdout:CentOS Stream 9 - CRB 7.6 MB/s | 6.2 MB 00:00 2024-05-08T06:19:01.569 INFO:teuthology.orchestra.run.smithi134.stdout:CentOS Stream 9 - Extras packages 61 kB/s | 16 kB 00:00 2024-05-08T06:19:02.696 INFO:teuthology.orchestra.run.smithi082.stdout:CentOS Stream 9 - Extras packages 13 kB/s | 16 kB 00:01 2024-05-08T06:19:03.806 INFO:teuthology.orchestra.run.smithi082.stdout:Extra Packages for Enterprise Linux 22 MB/s | 21 MB 00:00 2024-05-08T06:19:04.578 INFO:teuthology.orchestra.run.smithi134.stdout:Extra Packages for Enterprise Linux 7.4 MB/s | 21 MB 00:02 2024-05-08T06:19:08.956 INFO:teuthology.orchestra.run.smithi082.stdout:lab-extras 12 kB/s | 1.7 kB 00:00 2024-05-08T06:19:09.920 INFO:teuthology.orchestra.run.smithi134.stdout:lab-extras 13 kB/s | 1.7 kB 00:00 2024-05-08T06:19:10.962 INFO:teuthology.orchestra.run.smithi082.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-05-08T06:19:10.962 INFO:teuthology.orchestra.run.smithi082.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-05-08T06:19:11.046 INFO:teuthology.orchestra.run.smithi082.stdout:Dependencies resolved. 2024-05-08T06:19:11.051 INFO:teuthology.orchestra.run.smithi082.stdout:================================================================================ 2024-05-08T06:19:11.051 INFO:teuthology.orchestra.run.smithi082.stdout: Package Architecture Version Repository Size 2024-05-08T06:19:11.051 INFO:teuthology.orchestra.run.smithi082.stdout:================================================================================ 2024-05-08T06:19:11.051 INFO:teuthology.orchestra.run.smithi082.stdout:Installing: 2024-05-08T06:19:11.051 INFO:teuthology.orchestra.run.smithi082.stdout: kernel x86_64 5.14.0-444.el9 baseos 484 k 2024-05-08T06:19:11.051 INFO:teuthology.orchestra.run.smithi082.stdout:Installing dependencies: 2024-05-08T06:19:11.051 INFO:teuthology.orchestra.run.smithi082.stdout: kernel-core x86_64 5.14.0-444.el9 baseos 15 M 2024-05-08T06:19:11.051 INFO:teuthology.orchestra.run.smithi082.stdout: kernel-modules x86_64 5.14.0-444.el9 baseos 34 M 2024-05-08T06:19:11.051 INFO:teuthology.orchestra.run.smithi082.stdout: kernel-modules-core x86_64 5.14.0-444.el9 baseos 28 M 2024-05-08T06:19:11.051 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:19:11.052 INFO:teuthology.orchestra.run.smithi082.stdout:Transaction Summary 2024-05-08T06:19:11.052 INFO:teuthology.orchestra.run.smithi082.stdout:================================================================================ 2024-05-08T06:19:11.052 INFO:teuthology.orchestra.run.smithi082.stdout:Install 4 Packages 2024-05-08T06:19:11.052 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:19:11.052 INFO:teuthology.orchestra.run.smithi082.stdout:Total download size: 78 M 2024-05-08T06:19:11.052 INFO:teuthology.orchestra.run.smithi082.stdout:Installed size: 123 M 2024-05-08T06:19:11.052 INFO:teuthology.orchestra.run.smithi082.stdout:Downloading Packages: 2024-05-08T06:19:11.407 INFO:teuthology.orchestra.run.smithi082.stdout:(1/4): kernel-5.14.0-444.el9.x86_64.rpm 2.0 MB/s | 484 kB 00:00 2024-05-08T06:19:11.926 INFO:teuthology.orchestra.run.smithi134.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-05-08T06:19:11.926 INFO:teuthology.orchestra.run.smithi134.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-05-08T06:19:12.013 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-05-08T06:19:12.028 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-05-08T06:19:12.028 INFO:teuthology.orchestra.run.smithi134.stdout: Package Architecture Version Repository Size 2024-05-08T06:19:12.028 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-05-08T06:19:12.028 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2024-05-08T06:19:12.028 INFO:teuthology.orchestra.run.smithi134.stdout: kernel x86_64 5.14.0-444.el9 baseos 484 k 2024-05-08T06:19:12.028 INFO:teuthology.orchestra.run.smithi134.stdout:Installing dependencies: 2024-05-08T06:19:12.028 INFO:teuthology.orchestra.run.smithi134.stdout: kernel-core x86_64 5.14.0-444.el9 baseos 15 M 2024-05-08T06:19:12.028 INFO:teuthology.orchestra.run.smithi134.stdout: kernel-modules x86_64 5.14.0-444.el9 baseos 34 M 2024-05-08T06:19:12.028 INFO:teuthology.orchestra.run.smithi134.stdout: kernel-modules-core x86_64 5.14.0-444.el9 baseos 28 M 2024-05-08T06:19:12.028 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:19:12.029 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-05-08T06:19:12.029 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-05-08T06:19:12.029 INFO:teuthology.orchestra.run.smithi134.stdout:Install 4 Packages 2024-05-08T06:19:12.029 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:19:12.029 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 78 M 2024-05-08T06:19:12.029 INFO:teuthology.orchestra.run.smithi134.stdout:Installed size: 123 M 2024-05-08T06:19:12.030 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-05-08T06:19:12.746 INFO:teuthology.orchestra.run.smithi134.stdout:(1/4): kernel-5.14.0-444.el9.x86_64.rpm 718 kB/s | 484 kB 00:00 2024-05-08T06:19:13.582 INFO:teuthology.orchestra.run.smithi082.stdout:(2/4): kernel-core-5.14.0-444.el9.x86_64.rpm 6.4 MB/s | 15 MB 00:02 2024-05-08T06:19:14.613 INFO:teuthology.orchestra.run.smithi134.stdout:(2/4): kernel-core-5.14.0-444.el9.x86_64.rpm 6.1 MB/s | 15 MB 00:02 2024-05-08T06:19:15.673 INFO:teuthology.orchestra.run.smithi082.stdout:(3/4): kernel-modules-core-5.14.0-444.el9.x86_6 6.6 MB/s | 28 MB 00:04 2024-05-08T06:19:19.015 INFO:teuthology.orchestra.run.smithi082.stdout:(4/4): kernel-modules-5.14.0-444.el9.x86_64.rpm 4.3 MB/s | 34 MB 00:07 2024-05-08T06:19:19.015 INFO:teuthology.orchestra.run.smithi082.stdout:-------------------------------------------------------------------------------- 2024-05-08T06:19:19.015 INFO:teuthology.orchestra.run.smithi082.stdout:Total 9.8 MB/s | 78 MB 00:07 2024-05-08T06:19:19.697 INFO:teuthology.orchestra.run.smithi082.stdout:Running transaction check 2024-05-08T06:19:20.041 INFO:teuthology.orchestra.run.smithi082.stdout:Transaction check succeeded. 2024-05-08T06:19:20.041 INFO:teuthology.orchestra.run.smithi082.stdout:Running transaction test 2024-05-08T06:19:20.958 INFO:teuthology.orchestra.run.smithi082.stdout:Transaction test succeeded. 2024-05-08T06:19:20.958 INFO:teuthology.orchestra.run.smithi082.stdout:Running transaction 2024-05-08T06:19:23.358 INFO:teuthology.orchestra.run.smithi082.stdout: Preparing : 1/1 2024-05-08T06:19:24.165 INFO:teuthology.orchestra.run.smithi082.stdout: Installing : kernel-modules-core-5.14.0-444.el9.x86_64 1/4 2024-05-08T06:19:24.256 INFO:teuthology.orchestra.run.smithi082.stdout: Installing : kernel-core-5.14.0-444.el9.x86_64 2/4 2024-05-08T06:19:26.933 INFO:teuthology.orchestra.run.smithi082.stdout: Running scriptlet: kernel-core-5.14.0-444.el9.x86_64 2/4 2024-05-08T06:19:27.059 INFO:teuthology.orchestra.run.smithi082.stdout: Installing : kernel-modules-5.14.0-444.el9.x86_64 3/4 2024-05-08T06:19:31.772 INFO:teuthology.orchestra.run.smithi082.stdout: Running scriptlet: kernel-modules-5.14.0-444.el9.x86_64 3/4 2024-05-08T06:19:31.819 INFO:teuthology.orchestra.run.smithi082.stdout: Installing : kernel-5.14.0-444.el9.x86_64 4/4 2024-05-08T06:19:36.578 INFO:teuthology.orchestra.run.smithi082.stdout: Running scriptlet: kernel-modules-core-5.14.0-444.el9.x86_64 4/4 2024-05-08T06:19:42.977 INFO:teuthology.orchestra.run.smithi134.stdout:(3/4): kernel-modules-core-5.14.0-444.el9.x86_6 947 kB/s | 28 MB 00:30 2024-05-08T06:19:49.363 INFO:teuthology.orchestra.run.smithi134.stdout:(4/4): kernel-modules-5.14.0-444.el9.x86_64.rpm 932 kB/s | 34 MB 00:37 2024-05-08T06:19:49.363 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-05-08T06:19:49.363 INFO:teuthology.orchestra.run.smithi134.stdout:Total 2.1 MB/s | 78 MB 00:37 2024-05-08T06:19:50.037 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-05-08T06:19:50.371 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-05-08T06:19:50.371 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-05-08T06:19:51.308 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-05-08T06:19:51.308 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-05-08T06:19:53.715 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-05-08T06:19:54.442 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : kernel-modules-core-5.14.0-444.el9.x86_64 1/4 2024-05-08T06:19:54.517 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : kernel-core-5.14.0-444.el9.x86_64 2/4 2024-05-08T06:19:57.197 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: kernel-core-5.14.0-444.el9.x86_64 2/4 2024-05-08T06:19:57.346 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : kernel-modules-5.14.0-444.el9.x86_64 3/4 2024-05-08T06:20:02.034 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: kernel-modules-5.14.0-444.el9.x86_64 3/4 2024-05-08T06:20:02.146 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : kernel-5.14.0-444.el9.x86_64 4/4 2024-05-08T06:20:05.797 INFO:teuthology.orchestra.run.smithi082.stdout: Running scriptlet: kernel-core-5.14.0-444.el9.x86_64 4/4 2024-05-08T06:20:05.835 INFO:teuthology.orchestra.run.smithi082.stdout: Running scriptlet: kernel-modules-5.14.0-444.el9.x86_64 4/4 2024-05-08T06:20:06.119 INFO:teuthology.orchestra.run.smithi082.stdout: Running scriptlet: kernel-5.14.0-444.el9.x86_64 4/4 2024-05-08T06:20:06.119 INFO:teuthology.orchestra.run.smithi082.stdout: Verifying : kernel-5.14.0-444.el9.x86_64 1/4 2024-05-08T06:20:06.119 INFO:teuthology.orchestra.run.smithi082.stdout: Verifying : kernel-core-5.14.0-444.el9.x86_64 2/4 2024-05-08T06:20:06.119 INFO:teuthology.orchestra.run.smithi082.stdout: Verifying : kernel-modules-5.14.0-444.el9.x86_64 3/4 2024-05-08T06:20:06.873 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: kernel-modules-core-5.14.0-444.el9.x86_64 4/4 2024-05-08T06:20:06.894 INFO:teuthology.orchestra.run.smithi082.stdout: Verifying : kernel-modules-core-5.14.0-444.el9.x86_64 4/4 2024-05-08T06:20:06.894 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:20:06.894 INFO:teuthology.orchestra.run.smithi082.stdout:Installed: 2024-05-08T06:20:06.894 INFO:teuthology.orchestra.run.smithi082.stdout: kernel-5.14.0-444.el9.x86_64 2024-05-08T06:20:06.894 INFO:teuthology.orchestra.run.smithi082.stdout: kernel-core-5.14.0-444.el9.x86_64 2024-05-08T06:20:06.894 INFO:teuthology.orchestra.run.smithi082.stdout: kernel-modules-5.14.0-444.el9.x86_64 2024-05-08T06:20:06.894 INFO:teuthology.orchestra.run.smithi082.stdout: kernel-modules-core-5.14.0-444.el9.x86_64 2024-05-08T06:20:06.894 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:20:06.895 INFO:teuthology.orchestra.run.smithi082.stdout:Complete! 2024-05-08T06:20:07.211 DEBUG:teuthology.orchestra.run.smithi082:> rpm -q kernel | sort -rV | head -n 1 2024-05-08T06:20:07.300 INFO:teuthology.orchestra.run.smithi082.stdout:kernel-5.14.0-444.el9.x86_64 2024-05-08T06:20:07.301 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-444.el9.x86_64 2024-05-08T06:20:07.301 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-444.el9.x86_64 2024-05-08T06:20:07.301 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-05-08T06:20:07.301 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-05-08T06:20:07.301 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-05-08T06:20:07.301 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi082.front.sepia.ceph.com, path=None, version=distro) 2024-05-08T06:20:07.302 DEBUG:teuthology.orchestra.run.smithi082:> rpm -q kernel | sort -rV | head -n 1 2024-05-08T06:20:07.351 INFO:teuthology.orchestra.run.smithi082.stdout:kernel-5.14.0-444.el9.x86_64 2024-05-08T06:20:07.351 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-444.el9.x86_64 2024-05-08T06:20:07.351 DEBUG:teuthology.orchestra.run.smithi082:> sudo rpm -qi grub2-tools 2024-05-08T06:20:07.381 INFO:teuthology.orchestra.run.smithi082.stdout:Name : grub2-tools 2024-05-08T06:20:07.382 INFO:teuthology.orchestra.run.smithi082.stdout:Epoch : 1 2024-05-08T06:20:07.382 INFO:teuthology.orchestra.run.smithi082.stdout:Version : 2.06 2024-05-08T06:20:07.382 INFO:teuthology.orchestra.run.smithi082.stdout:Release : 68.el9 2024-05-08T06:20:07.382 INFO:teuthology.orchestra.run.smithi082.stdout:Architecture: x86_64 2024-05-08T06:20:07.382 INFO:teuthology.orchestra.run.smithi082.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-05-08T06:20:07.383 INFO:teuthology.orchestra.run.smithi082.stdout:Group : Unspecified 2024-05-08T06:20:07.383 INFO:teuthology.orchestra.run.smithi082.stdout:Size : 8264254 2024-05-08T06:20:07.383 INFO:teuthology.orchestra.run.smithi082.stdout:License : GPLv3+ 2024-05-08T06:20:07.383 INFO:teuthology.orchestra.run.smithi082.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-05-08T06:20:07.383 INFO:teuthology.orchestra.run.smithi082.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-05-08T06:20:07.383 INFO:teuthology.orchestra.run.smithi082.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-05-08T06:20:07.383 INFO:teuthology.orchestra.run.smithi082.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-05-08T06:20:07.383 INFO:teuthology.orchestra.run.smithi082.stdout:Packager : builder@centos.org 2024-05-08T06:20:07.383 INFO:teuthology.orchestra.run.smithi082.stdout:Vendor : CentOS 2024-05-08T06:20:07.383 INFO:teuthology.orchestra.run.smithi082.stdout:URL : http://www.gnu.org/software/grub/ 2024-05-08T06:20:07.383 INFO:teuthology.orchestra.run.smithi082.stdout:Summary : Support tools for GRUB. 2024-05-08T06:20:07.383 INFO:teuthology.orchestra.run.smithi082.stdout:Description : 2024-05-08T06:20:07.383 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:20:07.383 INFO:teuthology.orchestra.run.smithi082.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-05-08T06:20:07.383 INFO:teuthology.orchestra.run.smithi082.stdout:customizable bootloader with modular architecture. It supports a rich 2024-05-08T06:20:07.384 INFO:teuthology.orchestra.run.smithi082.stdout:variety of kernel formats, file systems, computer architectures and 2024-05-08T06:20:07.384 INFO:teuthology.orchestra.run.smithi082.stdout:hardware devices. 2024-05-08T06:20:07.384 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:20:07.384 INFO:teuthology.orchestra.run.smithi082.stdout:This subpackage provides tools for support of all platforms. 2024-05-08T06:20:07.385 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-05-08T06:20:07.385 INFO:teuthology.task.kernel:Updating grub on smithi082 to boot 5.14.0-444.el9.x86_64 2024-05-08T06:20:07.385 DEBUG:teuthology.orchestra.run.smithi082:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-05-08T06:20:07.995 INFO:teuthology.orchestra.run.smithi082.stderr:Generating grub configuration file ... 2024-05-08T06:20:09.373 INFO:teuthology.orchestra.run.smithi082.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-05-08T06:20:09.399 INFO:teuthology.orchestra.run.smithi082.stderr:done 2024-05-08T06:20:09.401 DEBUG:teuthology.orchestra.run.smithi082:> mktemp 2024-05-08T06:20:09.415 INFO:teuthology.orchestra.run.smithi082.stdout:/tmp/tmp.JlyxkDaMLL 2024-05-08T06:20:09.416 DEBUG:teuthology.orchestra.run.smithi082:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.JlyxkDaMLL 2024-05-08T06:20:09.480 DEBUG:teuthology.orchestra.run.smithi082:> sudo chmod 0666 /tmp/tmp.JlyxkDaMLL 2024-05-08T06:20:09.614 DEBUG:teuthology.orchestra.remote:smithi082:/tmp/tmp.JlyxkDaMLL is 6KB 2024-05-08T06:20:09.634 DEBUG:teuthology.orchestra.run.smithi082:> rm -fr /tmp/tmp.JlyxkDaMLL 2024-05-08T06:20:09.648 DEBUG:teuthology.orchestra.run.smithi082:> sudo /bin/ls /boot/loader/entries || true 2024-05-08T06:20:09.717 INFO:teuthology.orchestra.run.smithi082.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-05-08T06:20:09.717 INFO:teuthology.orchestra.run.smithi082.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-05-08T06:20:09.717 INFO:teuthology.orchestra.run.smithi082.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-05-08T06:20:09.717 INFO:teuthology.orchestra.run.smithi082.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-444.el9.x86_64.conf 2024-05-08T06:20:09.719 DEBUG:teuthology.orchestra.run.smithi082:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-444.el9.x86_64 2024-05-08T06:20:09.862 DEBUG:teuthology.orchestra.run.smithi082:> sudo shutdown -r now 2024-05-08T06:20:35.996 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: kernel-core-5.14.0-444.el9.x86_64 4/4 2024-05-08T06:20:36.033 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: kernel-modules-5.14.0-444.el9.x86_64 4/4 2024-05-08T06:20:36.424 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: kernel-5.14.0-444.el9.x86_64 4/4 2024-05-08T06:20:36.424 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : kernel-5.14.0-444.el9.x86_64 1/4 2024-05-08T06:20:36.424 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : kernel-core-5.14.0-444.el9.x86_64 2/4 2024-05-08T06:20:36.424 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : kernel-modules-5.14.0-444.el9.x86_64 3/4 2024-05-08T06:20:37.194 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : kernel-modules-core-5.14.0-444.el9.x86_64 4/4 2024-05-08T06:20:37.194 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:20:37.194 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-05-08T06:20:37.194 INFO:teuthology.orchestra.run.smithi134.stdout: kernel-5.14.0-444.el9.x86_64 2024-05-08T06:20:37.194 INFO:teuthology.orchestra.run.smithi134.stdout: kernel-core-5.14.0-444.el9.x86_64 2024-05-08T06:20:37.194 INFO:teuthology.orchestra.run.smithi134.stdout: kernel-modules-5.14.0-444.el9.x86_64 2024-05-08T06:20:37.194 INFO:teuthology.orchestra.run.smithi134.stdout: kernel-modules-core-5.14.0-444.el9.x86_64 2024-05-08T06:20:37.194 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:20:37.195 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-05-08T06:20:37.574 DEBUG:teuthology.orchestra.run.smithi134:> rpm -q kernel | sort -rV | head -n 1 2024-05-08T06:20:37.645 INFO:teuthology.orchestra.run.smithi134.stdout:kernel-5.14.0-444.el9.x86_64 2024-05-08T06:20:37.646 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-444.el9.x86_64 2024-05-08T06:20:37.646 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-444.el9.x86_64 2024-05-08T06:20:37.646 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-05-08T06:20:37.646 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-05-08T06:20:37.646 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-05-08T06:20:37.646 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi134.front.sepia.ceph.com, path=None, version=distro) 2024-05-08T06:20:37.647 DEBUG:teuthology.orchestra.run.smithi134:> rpm -q kernel | sort -rV | head -n 1 2024-05-08T06:20:37.696 INFO:teuthology.orchestra.run.smithi134.stdout:kernel-5.14.0-444.el9.x86_64 2024-05-08T06:20:37.696 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-444.el9.x86_64 2024-05-08T06:20:37.696 DEBUG:teuthology.orchestra.run.smithi134:> sudo rpm -qi grub2-tools 2024-05-08T06:20:37.728 INFO:teuthology.orchestra.run.smithi134.stdout:Name : grub2-tools 2024-05-08T06:20:37.728 INFO:teuthology.orchestra.run.smithi134.stdout:Epoch : 1 2024-05-08T06:20:37.728 INFO:teuthology.orchestra.run.smithi134.stdout:Version : 2.06 2024-05-08T06:20:37.728 INFO:teuthology.orchestra.run.smithi134.stdout:Release : 68.el9 2024-05-08T06:20:37.728 INFO:teuthology.orchestra.run.smithi134.stdout:Architecture: x86_64 2024-05-08T06:20:37.728 INFO:teuthology.orchestra.run.smithi134.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-05-08T06:20:37.728 INFO:teuthology.orchestra.run.smithi134.stdout:Group : Unspecified 2024-05-08T06:20:37.729 INFO:teuthology.orchestra.run.smithi134.stdout:Size : 8264254 2024-05-08T06:20:37.729 INFO:teuthology.orchestra.run.smithi134.stdout:License : GPLv3+ 2024-05-08T06:20:37.729 INFO:teuthology.orchestra.run.smithi134.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-05-08T06:20:37.729 INFO:teuthology.orchestra.run.smithi134.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-05-08T06:20:37.729 INFO:teuthology.orchestra.run.smithi134.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-05-08T06:20:37.729 INFO:teuthology.orchestra.run.smithi134.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-05-08T06:20:37.729 INFO:teuthology.orchestra.run.smithi134.stdout:Packager : builder@centos.org 2024-05-08T06:20:37.729 INFO:teuthology.orchestra.run.smithi134.stdout:Vendor : CentOS 2024-05-08T06:20:37.729 INFO:teuthology.orchestra.run.smithi134.stdout:URL : http://www.gnu.org/software/grub/ 2024-05-08T06:20:37.729 INFO:teuthology.orchestra.run.smithi134.stdout:Summary : Support tools for GRUB. 2024-05-08T06:20:37.729 INFO:teuthology.orchestra.run.smithi134.stdout:Description : 2024-05-08T06:20:37.729 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:20:37.729 INFO:teuthology.orchestra.run.smithi134.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-05-08T06:20:37.729 INFO:teuthology.orchestra.run.smithi134.stdout:customizable bootloader with modular architecture. It supports a rich 2024-05-08T06:20:37.730 INFO:teuthology.orchestra.run.smithi134.stdout:variety of kernel formats, file systems, computer architectures and 2024-05-08T06:20:37.730 INFO:teuthology.orchestra.run.smithi134.stdout:hardware devices. 2024-05-08T06:20:37.730 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:20:37.730 INFO:teuthology.orchestra.run.smithi134.stdout:This subpackage provides tools for support of all platforms. 2024-05-08T06:20:37.731 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-05-08T06:20:37.731 INFO:teuthology.task.kernel:Updating grub on smithi134 to boot 5.14.0-444.el9.x86_64 2024-05-08T06:20:37.731 DEBUG:teuthology.orchestra.run.smithi134:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-05-08T06:20:38.353 INFO:teuthology.orchestra.run.smithi134.stderr:Generating grub configuration file ... 2024-05-08T06:20:39.852 INFO:teuthology.orchestra.run.smithi134.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-05-08T06:20:39.865 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-05-08T06:20:39.865 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi082.front.sepia.ceph.com' 2024-05-08T06:20:39.866 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi082.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-08T06:20:39.881 INFO:teuthology.orchestra.run.smithi134.stderr:done 2024-05-08T06:20:39.883 DEBUG:teuthology.orchestra.run.smithi134:> mktemp 2024-05-08T06:20:39.898 INFO:teuthology.orchestra.run.smithi134.stdout:/tmp/tmp.uGL7R4d5sZ 2024-05-08T06:20:39.899 DEBUG:teuthology.orchestra.run.smithi134:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.uGL7R4d5sZ 2024-05-08T06:20:39.967 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod 0666 /tmp/tmp.uGL7R4d5sZ 2024-05-08T06:20:40.105 DEBUG:teuthology.orchestra.remote:smithi134:/tmp/tmp.uGL7R4d5sZ is 6KB 2024-05-08T06:20:40.154 DEBUG:teuthology.orchestra.run.smithi134:> rm -fr /tmp/tmp.uGL7R4d5sZ 2024-05-08T06:20:40.168 DEBUG:teuthology.orchestra.run.smithi134:> sudo /bin/ls /boot/loader/entries || true 2024-05-08T06:20:40.229 INFO:teuthology.orchestra.run.smithi134.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-05-08T06:20:40.230 INFO:teuthology.orchestra.run.smithi134.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-05-08T06:20:40.230 INFO:teuthology.orchestra.run.smithi134.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-05-08T06:20:40.230 INFO:teuthology.orchestra.run.smithi134.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-444.el9.x86_64.conf 2024-05-08T06:20:40.231 DEBUG:teuthology.orchestra.run.smithi134:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-444.el9.x86_64 2024-05-08T06:20:40.366 DEBUG:teuthology.orchestra.run.smithi134:> sudo shutdown -r now 2024-05-08T06:21:10.395 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-05-08T06:21:10.396 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-05-08T06:21:10.397 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-08T06:21:39.895 DEBUG:teuthology.orchestra.remote:timed out 2024-05-08T06:21:48.903 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi082.front.sepia.ceph.com' 2024-05-08T06:21:48.904 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi082.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-08T06:21:52.227 DEBUG:teuthology.orchestra.run.smithi082:> true 2024-05-08T06:21:52.752 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi082.front.sepia.ceph.com' 2024-05-08T06:21:52.752 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-444.el9.x86_64"... 2024-05-08T06:21:52.752 DEBUG:teuthology.orchestra.run.smithi082:> uname -r 2024-05-08T06:21:52.768 INFO:teuthology.orchestra.run.smithi082.stdout:5.14.0-444.el9.x86_64 2024-05-08T06:21:52.768 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-444.el9.x86_64 vs 5.14.0-444.el9.x86_64 2024-05-08T06:21:52.768 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-05-08T06:21:52.768 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-05-08T06:21:53.769 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-05-08T06:21:53.769 DEBUG:teuthology.orchestra.run.smithi082:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-05-08T06:21:53.853 INFO:teuthology.orchestra.run.smithi082.stdout:ttyS1 2024-05-08T06:21:53.910 DEBUG:teuthology.parallel:result is None 2024-05-08T06:22:10.399 DEBUG:teuthology.orchestra.remote:timed out 2024-05-08T06:22:19.400 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-05-08T06:22:19.401 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-08T06:22:22.852 DEBUG:teuthology.orchestra.run.smithi134:> true 2024-05-08T06:22:23.283 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-05-08T06:22:23.283 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-444.el9.x86_64"... 2024-05-08T06:22:23.284 DEBUG:teuthology.orchestra.run.smithi134:> uname -r 2024-05-08T06:22:23.298 INFO:teuthology.orchestra.run.smithi134.stdout:5.14.0-444.el9.x86_64 2024-05-08T06:22:23.298 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-444.el9.x86_64 vs 5.14.0-444.el9.x86_64 2024-05-08T06:22:23.298 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-05-08T06:22:23.299 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-05-08T06:22:24.299 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-05-08T06:22:24.300 DEBUG:teuthology.orchestra.run.smithi134:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-05-08T06:22:24.379 INFO:teuthology.orchestra.run.smithi134.stdout:ttyS1 2024-05-08T06:22:24.421 DEBUG:teuthology.parallel:result is None 2024-05-08T06:22:24.422 INFO:teuthology.run_tasks:Running task internal.base... 2024-05-08T06:22:24.429 INFO:teuthology.task.internal:Creating test directory... 2024-05-08T06:22:24.430 DEBUG:teuthology.orchestra.run.smithi082:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-05-08T06:22:24.433 DEBUG:teuthology.orchestra.run.smithi134:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-05-08T06:22:24.450 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-05-08T06:22:24.456 INFO:teuthology.run_tasks:Running task internal.archive... 2024-05-08T06:22:24.467 INFO:teuthology.task.internal:Creating archive directory... 2024-05-08T06:22:24.467 DEBUG:teuthology.orchestra.run.smithi082:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-05-08T06:22:24.494 DEBUG:teuthology.orchestra.run.smithi134:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-05-08T06:22:24.624 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-05-08T06:22:24.631 INFO:teuthology.task.internal:Enabling coredump saving... 2024-05-08T06:22:24.631 DEBUG:teuthology.orchestra.run.smithi082:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-05-08T06:22:24.634 DEBUG:teuthology.orchestra.run.smithi134:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-05-08T06:22:24.669 INFO:teuthology.orchestra.run.smithi134.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-08T06:22:24.669 INFO:teuthology.orchestra.run.smithi082.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-08T06:22:24.679 INFO:teuthology.orchestra.run.smithi134.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-08T06:22:24.680 INFO:teuthology.orchestra.run.smithi082.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-08T06:22:24.682 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-05-08T06:22:24.689 INFO:teuthology.task.internal:Configuring sudo... 2024-05-08T06:22:24.689 DEBUG:teuthology.orchestra.run.smithi082:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-05-08T06:22:24.725 DEBUG:teuthology.orchestra.run.smithi134:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-05-08T06:22:24.750 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-05-08T06:22:24.759 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-05-08T06:22:24.760 DEBUG:teuthology.orchestra.run.smithi082:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-05-08T06:22:24.793 DEBUG:teuthology.orchestra.run.smithi134:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-05-08T06:22:24.808 DEBUG:teuthology.orchestra.run.smithi082:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-08T06:22:24.894 DEBUG:teuthology.orchestra.run.smithi082:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-08T06:22:24.939 DEBUG:teuthology.orchestra.run.smithi082:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-08T06:22:25.027 DEBUG:teuthology.orchestra.run.smithi082:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-08T06:22:25.064 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2024-05-08T06:22:25.064 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-05-08T06:22:25.207 DEBUG:teuthology.orchestra.run.smithi134:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-08T06:22:25.256 DEBUG:teuthology.orchestra.run.smithi134:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-08T06:22:25.296 DEBUG:teuthology.orchestra.run.smithi134:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-08T06:22:25.380 DEBUG:teuthology.orchestra.run.smithi134:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-08T06:22:25.419 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-05-08T06:22:25.419 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-05-08T06:22:25.486 DEBUG:teuthology.orchestra.run.smithi082:> sudo service rsyslog restart 2024-05-08T06:22:25.488 DEBUG:teuthology.orchestra.run.smithi134:> sudo service rsyslog restart 2024-05-08T06:22:25.547 INFO:teuthology.orchestra.run.smithi082.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-08T06:22:25.562 INFO:teuthology.orchestra.run.smithi134.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-08T06:22:26.055 INFO:teuthology.run_tasks:Running task internal.timer... 2024-05-08T06:22:26.063 INFO:teuthology.task.internal:Starting timer... 2024-05-08T06:22:26.063 INFO:teuthology.run_tasks:Running task pcp... 2024-05-08T06:22:26.074 INFO:teuthology.run_tasks:Running task selinux... 2024-05-08T06:22:26.082 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-05-08T06:22:26.082 DEBUG:teuthology.orchestra.run.smithi082:> sudo service auditd rotate 2024-05-08T06:22:26.157 INFO:teuthology.orchestra.run.smithi082.stdout:Rotating logs: 2024-05-08T06:22:26.159 DEBUG:teuthology.orchestra.run.smithi134:> sudo service auditd rotate 2024-05-08T06:22:26.243 INFO:teuthology.orchestra.run.smithi134.stdout:Rotating logs: 2024-05-08T06:22:26.245 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-05-08T06:22:26.245 DEBUG:teuthology.orchestra.run.smithi082:> /usr/sbin/getenforce 2024-05-08T06:22:26.277 INFO:teuthology.orchestra.run.smithi082.stdout:Permissive 2024-05-08T06:22:26.277 DEBUG:teuthology.orchestra.run.smithi134:> /usr/sbin/getenforce 2024-05-08T06:22:26.295 INFO:teuthology.orchestra.run.smithi134.stdout:Permissive 2024-05-08T06:22:26.295 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi082.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi134.front.sepia.ceph.com': 'permissive'} 2024-05-08T06:22:26.295 DEBUG:teuthology.orchestra.run.smithi082:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-05-08T06:22:26.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-08T06:22:26.346 DEBUG:teuthology.orchestra.run.smithi134:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-05-08T06:22:26.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-08T06:22:26.371 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-05-08T06:22:26.371 DEBUG:teuthology.orchestra.run.smithi082:> sudo /usr/sbin/setenforce permissive 2024-05-08T06:22:26.413 DEBUG:teuthology.orchestra.run.smithi134:> sudo /usr/sbin/setenforce permissive 2024-05-08T06:22:26.496 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-05-08T06:22:26.506 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-05-08T06:22:26.525 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-05-08T06:22:26.527 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi082.front.sepia.ceph.com,smithi134.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-05-08T06:27:13.160 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi082.front.sepia.ceph.com'), Remote(name='ubuntu@smithi134.front.sepia.ceph.com')] 2024-05-08T06:27:13.161 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi082.front.sepia.ceph.com' 2024-05-08T06:27:13.163 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi082.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-08T06:27:13.243 DEBUG:teuthology.orchestra.run.smithi082:> true 2024-05-08T06:27:13.326 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi082.front.sepia.ceph.com' 2024-05-08T06:27:13.326 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-05-08T06:27:13.327 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-08T06:27:13.404 DEBUG:teuthology.orchestra.run.smithi134:> true 2024-05-08T06:27:13.482 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-05-08T06:27:13.483 INFO:teuthology.run_tasks:Running task clock... 2024-05-08T06:27:13.497 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-05-08T06:27:13.497 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-05-08T06:27:13.497 DEBUG:teuthology.orchestra.run.smithi082:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-08T06:27:13.500 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-05-08T06:27:13.500 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-08T06:27:13.527 INFO:teuthology.orchestra.run.smithi082.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-05-08T06:27:13.541 INFO:teuthology.orchestra.run.smithi082.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-05-08T06:27:13.552 INFO:teuthology.orchestra.run.smithi134.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-05-08T06:27:13.564 INFO:teuthology.orchestra.run.smithi082.stderr:sudo: ntpd: command not found 2024-05-08T06:27:13.567 INFO:teuthology.orchestra.run.smithi134.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-05-08T06:27:13.575 INFO:teuthology.orchestra.run.smithi082.stdout:506 Cannot talk to daemon 2024-05-08T06:27:13.589 INFO:teuthology.orchestra.run.smithi082.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-05-08T06:27:13.593 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: ntpd: command not found 2024-05-08T06:27:13.603 INFO:teuthology.orchestra.run.smithi082.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-05-08T06:27:13.604 INFO:teuthology.orchestra.run.smithi134.stdout:506 Cannot talk to daemon 2024-05-08T06:27:13.618 INFO:teuthology.orchestra.run.smithi134.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-05-08T06:27:13.634 INFO:teuthology.orchestra.run.smithi134.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-05-08T06:27:13.658 INFO:teuthology.orchestra.run.smithi082.stderr:bash: line 1: ntpq: command not found 2024-05-08T06:27:13.662 INFO:teuthology.orchestra.run.smithi082.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-08T06:27:13.662 INFO:teuthology.orchestra.run.smithi082.stdout:=============================================================================== 2024-05-08T06:27:13.663 INFO:teuthology.orchestra.run.smithi082.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-08T06:27:13.663 INFO:teuthology.orchestra.run.smithi082.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-08T06:27:13.663 INFO:teuthology.orchestra.run.smithi082.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-08T06:27:13.663 INFO:teuthology.orchestra.run.smithi082.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-08T06:27:13.688 INFO:teuthology.orchestra.run.smithi134.stderr:bash: line 1: ntpq: command not found 2024-05-08T06:27:13.691 INFO:teuthology.orchestra.run.smithi134.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-08T06:27:13.692 INFO:teuthology.orchestra.run.smithi134.stdout:=============================================================================== 2024-05-08T06:27:13.692 INFO:teuthology.orchestra.run.smithi134.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-08T06:27:13.692 INFO:teuthology.orchestra.run.smithi134.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-08T06:27:13.692 INFO:teuthology.orchestra.run.smithi134.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-08T06:27:13.692 INFO:teuthology.orchestra.run.smithi134.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-08T06:27:13.693 INFO:teuthology.run_tasks:Running task cephadm... 2024-05-08T06:27:13.793 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': '7c8f650b36e258f639fa4a83becade57cbfd2009'} 2024-05-08T06:27:13.793 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-05-08T06:27:13.794 INFO:tasks.cephadm:Cluster fsid is 02181fb0-0d04-11ef-bc97-c7b262605968 2024-05-08T06:27:13.794 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-05-08T06:27:13.794 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-05-08T06:27:13.794 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi082': '172.21.15.82', 'mon.smithi134': '172.21.15.134'} 2024-05-08T06:27:13.794 INFO:tasks.cephadm:Normalizing hostnames... 2024-05-08T06:27:13.794 DEBUG:teuthology.orchestra.run.smithi082:> sudo hostname $(hostname -s) 2024-05-08T06:27:13.820 DEBUG:teuthology.orchestra.run.smithi134:> sudo hostname $(hostname -s) 2024-05-08T06:27:13.847 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-05-08T06:27:13.847 DEBUG:teuthology.orchestra.run.smithi082:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-05-08T06:27:14.159 INFO:teuthology.orchestra.run.smithi082.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 May 8 06:27 /home/ubuntu/cephtest/cephadm 2024-05-08T06:27:14.160 DEBUG:teuthology.orchestra.run.smithi134:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-05-08T06:27:14.334 INFO:teuthology.orchestra.run.smithi134.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 May 8 06:27 /home/ubuntu/cephtest/cephadm 2024-05-08T06:27:14.334 DEBUG:teuthology.orchestra.run.smithi082:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-05-08T06:27:14.351 DEBUG:teuthology.orchestra.run.smithi134:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-05-08T06:27:14.385 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-05-08T06:27:14.385 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-05-08T06:27:14.399 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-05-08T06:27:15.910 INFO:teuthology.orchestra.run.smithi134.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-08T06:27:15.999 INFO:teuthology.orchestra.run.smithi082.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-08T06:27:34.507 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2024-05-08T06:27:34.508 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-05-08T06:27:34.508 INFO:teuthology.orchestra.run.smithi082.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-05-08T06:27:34.508 INFO:teuthology.orchestra.run.smithi082.stdout: "repo_digests": [ 2024-05-08T06:27:34.508 INFO:teuthology.orchestra.run.smithi082.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-05-08T06:27:34.508 INFO:teuthology.orchestra.run.smithi082.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-05-08T06:27:34.508 INFO:teuthology.orchestra.run.smithi082.stdout: ] 2024-05-08T06:27:34.508 INFO:teuthology.orchestra.run.smithi082.stdout:} 2024-05-08T06:27:35.241 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-05-08T06:27:35.242 INFO:teuthology.orchestra.run.smithi134.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-05-08T06:27:35.242 INFO:teuthology.orchestra.run.smithi134.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-05-08T06:27:35.242 INFO:teuthology.orchestra.run.smithi134.stdout: "repo_digests": [ 2024-05-08T06:27:35.242 INFO:teuthology.orchestra.run.smithi134.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-05-08T06:27:35.242 INFO:teuthology.orchestra.run.smithi134.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-05-08T06:27:35.242 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-05-08T06:27:35.242 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-05-08T06:27:35.262 DEBUG:teuthology.orchestra.run.smithi082:> sudo mkdir -p /etc/ceph 2024-05-08T06:27:35.292 DEBUG:teuthology.orchestra.run.smithi134:> sudo mkdir -p /etc/ceph 2024-05-08T06:27:35.322 DEBUG:teuthology.orchestra.run.smithi082:> sudo chmod 777 /etc/ceph 2024-05-08T06:27:35.358 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod 777 /etc/ceph 2024-05-08T06:27:35.388 INFO:tasks.cephadm:Writing seed config... 2024-05-08T06:27:35.390 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-05-08T06:27:35.390 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-05-08T06:27:35.390 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-05-08T06:27:35.390 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-05-08T06:27:35.390 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-05-08T06:27:35.390 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-05-08T06:27:35.390 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-05-08T06:27:35.390 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-05-08T06:27:35.391 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2024-05-08T06:27:35.391 DEBUG:teuthology.orchestra.run.smithi082:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-05-08T06:27:35.414 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 02181fb0-0d04-11ef-bc97-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-05-08T06:27:35.414 DEBUG:teuthology.orchestra.run.smithi082:mon.smithi082> sudo journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@mon.smithi082.service 2024-05-08T06:27:35.456 INFO:tasks.cephadm:Bootstrapping... 2024-05-08T06:27:35.457 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.82 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-05-08T06:27:35.649 INFO:teuthology.orchestra.run.smithi082.stderr:-------------------------------------------------------------------------------- 2024-05-08T06:27:35.649 INFO:teuthology.orchestra.run.smithi082.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '02181fb0-0d04-11ef-bc97-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.82', '--skip-admin-label'] 2024-05-08T06:27:35.673 INFO:teuthology.orchestra.run.smithi082.stderr:/bin/podman: 4.9.4-dev 2024-05-08T06:27:35.674 INFO:teuthology.orchestra.run.smithi082.stderr:Verifying podman|docker is present... 2024-05-08T06:27:35.696 INFO:teuthology.orchestra.run.smithi082.stderr:/bin/podman: 4.9.4-dev 2024-05-08T06:27:35.699 INFO:teuthology.orchestra.run.smithi082.stderr:Verifying lvm2 is present... 2024-05-08T06:27:35.699 INFO:teuthology.orchestra.run.smithi082.stderr:Verifying time synchronization is in place... 2024-05-08T06:27:35.706 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-05-08T06:27:35.713 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: inactive 2024-05-08T06:27:35.720 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: enabled 2024-05-08T06:27:35.726 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: active 2024-05-08T06:27:35.727 INFO:teuthology.orchestra.run.smithi082.stderr:Unit chronyd.service is enabled and running 2024-05-08T06:27:35.727 INFO:teuthology.orchestra.run.smithi082.stderr:Repeating the final host check... 2024-05-08T06:27:35.748 INFO:teuthology.orchestra.run.smithi082.stderr:/bin/podman: 4.9.4-dev 2024-05-08T06:27:35.750 INFO:teuthology.orchestra.run.smithi082.stderr:podman (/bin/podman) version 4.9.4 is present 2024-05-08T06:27:35.751 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl is present 2024-05-08T06:27:35.751 INFO:teuthology.orchestra.run.smithi082.stderr:lvcreate is present 2024-05-08T06:27:35.757 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-05-08T06:27:35.763 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: inactive 2024-05-08T06:27:35.770 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: enabled 2024-05-08T06:27:35.776 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: active 2024-05-08T06:27:35.777 INFO:teuthology.orchestra.run.smithi082.stderr:Unit chronyd.service is enabled and running 2024-05-08T06:27:35.777 INFO:teuthology.orchestra.run.smithi082.stderr:Host looks OK 2024-05-08T06:27:35.777 INFO:teuthology.orchestra.run.smithi082.stderr:Cluster fsid: 02181fb0-0d04-11ef-bc97-c7b262605968 2024-05-08T06:27:35.778 INFO:teuthology.orchestra.run.smithi082.stderr:Acquiring lock 140270798660320 on /run/cephadm/02181fb0-0d04-11ef-bc97-c7b262605968.lock 2024-05-08T06:27:35.778 INFO:teuthology.orchestra.run.smithi082.stderr:Lock 140270798660320 acquired on /run/cephadm/02181fb0-0d04-11ef-bc97-c7b262605968.lock 2024-05-08T06:27:35.778 INFO:teuthology.orchestra.run.smithi082.stderr:Verifying IP 172.21.15.82 port 3300 ... 2024-05-08T06:27:35.778 INFO:teuthology.orchestra.run.smithi082.stderr:Verifying IP 172.21.15.82 port 6789 ... 2024-05-08T06:27:35.778 INFO:teuthology.orchestra.run.smithi082.stderr:Base mon IP is 172.21.15.82, final addrv is [v2:172.21.15.82:3300,v1:172.21.15.82:6789] 2024-05-08T06:27:35.781 INFO:teuthology.orchestra.run.smithi082.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.82 metric 100 2024-05-08T06:27:35.781 INFO:teuthology.orchestra.run.smithi082.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.82 metric 100 2024-05-08T06:27:35.784 INFO:teuthology.orchestra.run.smithi082.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-05-08T06:27:35.785 INFO:teuthology.orchestra.run.smithi082.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-05-08T06:27:35.785 INFO:teuthology.orchestra.run.smithi082.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1748sec hoplimit 64 pref medium 2024-05-08T06:27:35.787 INFO:teuthology.orchestra.run.smithi082.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-05-08T06:27:35.787 INFO:teuthology.orchestra.run.smithi082.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-05-08T06:27:35.787 INFO:teuthology.orchestra.run.smithi082.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-05-08T06:27:35.787 INFO:teuthology.orchestra.run.smithi082.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-05-08T06:27:35.787 INFO:teuthology.orchestra.run.smithi082.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d3ff/64 scope link 2024-05-08T06:27:35.787 INFO:teuthology.orchestra.run.smithi082.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-05-08T06:27:35.788 INFO:teuthology.orchestra.run.smithi082.stderr:Mon IP `172.21.15.82` is in CIDR network `172.21.0.0/20` 2024-05-08T06:27:35.788 INFO:teuthology.orchestra.run.smithi082.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-05-08T06:27:35.788 INFO:teuthology.orchestra.run.smithi082.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-08T06:27:35.890 INFO:teuthology.orchestra.run.smithi082.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-05-08T06:27:36.353 INFO:teuthology.orchestra.run.smithi082.stderr:/bin/podman: Getting image source signatures 2024-05-08T06:27:36.353 INFO:teuthology.orchestra.run.smithi082.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-05-08T06:27:36.353 INFO:teuthology.orchestra.run.smithi082.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-05-08T06:27:36.353 INFO:teuthology.orchestra.run.smithi082.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-05-08T06:27:36.354 INFO:teuthology.orchestra.run.smithi082.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-05-08T06:27:36.354 INFO:teuthology.orchestra.run.smithi082.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-05-08T06:27:36.354 INFO:teuthology.orchestra.run.smithi082.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-05-08T06:27:37.495 INFO:teuthology.orchestra.run.smithi082.stderr:/bin/podman: Writing manifest to image destination 2024-05-08T06:27:38.670 INFO:teuthology.orchestra.run.smithi082.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-05-08T06:27:42.268 INFO:teuthology.orchestra.run.smithi082.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-05-08T06:27:47.758 INFO:teuthology.orchestra.run.smithi082.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-05-08T06:27:47.758 INFO:teuthology.orchestra.run.smithi082.stderr:Extracting ceph user uid/gid from container image... 2024-05-08T06:27:48.483 INFO:teuthology.orchestra.run.smithi082.stderr:stat: 167 167 2024-05-08T06:27:49.074 INFO:teuthology.orchestra.run.smithi082.stderr:Creating initial keys... 2024-05-08T06:27:49.397 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-authtool: AQBlGztmUKtCFRAA4Tbmmt1pkqX7obLXSGlpSw== 2024-05-08T06:27:50.361 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-authtool: AQBmGztm3Q6HEBAA1NOA+obXWnWOQl2dUzOHDg== 2024-05-08T06:27:51.341 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph-authtool: AQBnGztm5GlyEBAAQXNVTCO71GJ5V5jjHptfqQ== 2024-05-08T06:27:51.866 INFO:teuthology.orchestra.run.smithi082.stderr:Creating initial monmap... 2024-05-08T06:27:52.282 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-05-08T06:27:52.283 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-05-08T06:27:52.283 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 02181fb0-0d04-11ef-bc97-c7b262605968 2024-05-08T06:27:52.283 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-05-08T06:27:52.779 INFO:teuthology.orchestra.run.smithi082.stderr:monmaptool for smithi082 [v2:172.21.15.82:3300,v1:172.21.15.82:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-05-08T06:27:52.780 INFO:teuthology.orchestra.run.smithi082.stderr:setting min_mon_release = octopus 2024-05-08T06:27:52.781 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/monmaptool: set fsid to 02181fb0-0d04-11ef-bc97-c7b262605968 2024-05-08T06:27:52.781 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-05-08T06:27:52.781 INFO:teuthology.orchestra.run.smithi082.stderr: 2024-05-08T06:27:52.781 INFO:teuthology.orchestra.run.smithi082.stderr:Creating mon... 2024-05-08T06:27:54.415 INFO:teuthology.orchestra.run.smithi082.stderr:create mon.smithi082 on 2024-05-08T06:27:54.643 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-05-08T06:27:54.792 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-02181fb0-0d04-11ef-bc97-c7b262605968.target → /etc/systemd/system/ceph-02181fb0-0d04-11ef-bc97-c7b262605968.target. 2024-05-08T06:27:54.792 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-02181fb0-0d04-11ef-bc97-c7b262605968.target → /etc/systemd/system/ceph-02181fb0-0d04-11ef-bc97-c7b262605968.target. 2024-05-08T06:27:55.101 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Failed to reset failed state of unit ceph-02181fb0-0d04-11ef-bc97-c7b262605968@mon.smithi082.service: Unit ceph-02181fb0-0d04-11ef-bc97-c7b262605968@mon.smithi082.service not loaded. 2024-05-08T06:27:55.109 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Created symlink /etc/systemd/system/ceph-02181fb0-0d04-11ef-bc97-c7b262605968.target.wants/ceph-02181fb0-0d04-11ef-bc97-c7b262605968@mon.smithi082.service → /etc/systemd/system/ceph-02181fb0-0d04-11ef-bc97-c7b262605968@.service. 2024-05-08T06:27:55.813 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: disabled 2024-05-08T06:27:55.819 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: inactive 2024-05-08T06:27:55.820 INFO:teuthology.orchestra.run.smithi082.stderr:firewalld.service is not enabled 2024-05-08T06:27:55.820 INFO:teuthology.orchestra.run.smithi082.stderr:Not possible to enable service . firewalld.service is not available 2024-05-08T06:27:55.820 INFO:teuthology.orchestra.run.smithi082.stderr:Waiting for mon to start... 2024-05-08T06:27:55.820 INFO:teuthology.orchestra.run.smithi082.stderr:Waiting for mon... 2024-05-08T06:27:55.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:27:55 smithi082 systemd[1]: Started Ceph mon.smithi082 for 02181fb0-0d04-11ef-bc97-c7b262605968. 2024-05-08T06:27:56.317 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: cluster: 2024-05-08T06:27:56.317 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: id: 02181fb0-0d04-11ef-bc97-c7b262605968 2024-05-08T06:27:56.318 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-05-08T06:27:56.318 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2024-05-08T06:27:56.318 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: services: 2024-05-08T06:27:56.318 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi082 (age 0.575581s) 2024-05-08T06:27:56.318 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mgr: no daemons active 2024-05-08T06:27:56.318 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-05-08T06:27:56.318 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2024-05-08T06:27:56.318 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: data: 2024-05-08T06:27:56.318 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-05-08T06:27:56.318 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-05-08T06:27:56.318 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-05-08T06:27:56.318 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: pgs: 2024-05-08T06:27:56.318 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2024-05-08T06:27:56.881 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:27:56 smithi082 ceph-mon[22605]: mon.smithi082 is new leader, mons smithi082 in quorum (ranks 0) 2024-05-08T06:27:56.882 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:27:56 smithi082 ceph-mon[22605]: monmap e1: 1 mons at {smithi082=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0]} 2024-05-08T06:27:56.882 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:27:56 smithi082 ceph-mon[22605]: fsmap 2024-05-08T06:27:56.882 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:27:56 smithi082 ceph-mon[22605]: osdmap e1: 0 total, 0 up, 0 in 2024-05-08T06:27:56.882 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:27:56 smithi082 ceph-mon[22605]: mgrmap e1: no daemons active 2024-05-08T06:27:56.882 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:27:56 smithi082 ceph-mon[22605]: from='client.? 172.21.15.82:0/2933929456' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-08T06:27:56.884 INFO:teuthology.orchestra.run.smithi082.stderr:mon is available 2024-05-08T06:27:56.884 INFO:teuthology.orchestra.run.smithi082.stderr:Assimilating anything we can from ceph.conf... 2024-05-08T06:27:57.431 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2024-05-08T06:27:57.431 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: [global] 2024-05-08T06:27:57.431 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: fsid = 02181fb0-0d04-11ef-bc97-c7b262605968 2024-05-08T06:27:57.432 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.82:3300,v1:172.21.15.82:6789] 2024-05-08T06:27:57.432 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-05-08T06:27:57.432 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-05-08T06:27:57.432 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-05-08T06:27:57.432 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-05-08T06:27:57.432 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2024-05-08T06:27:57.432 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: [mgr] 2024-05-08T06:27:57.432 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-05-08T06:27:57.432 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2024-05-08T06:27:57.432 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: [osd] 2024-05-08T06:27:57.432 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-05-08T06:27:57.432 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-05-08T06:27:58.021 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:27:57 smithi082 ceph-mon[22605]: from='client.? 172.21.15.82:0/4141724392' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-05-08T06:27:58.021 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:27:57 smithi082 ceph-mon[22605]: from='client.? 172.21.15.82:0/4141724392' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-05-08T06:27:58.023 INFO:teuthology.orchestra.run.smithi082.stderr:Generating new minimal ceph.conf... 2024-05-08T06:27:59.011 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:27:58 smithi082 ceph-mon[22605]: from='client.? 172.21.15.82:0/1703305709' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:27:59.014 INFO:teuthology.orchestra.run.smithi082.stderr:Restarting the monitor... 2024-05-08T06:27:59.262 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:27:59 smithi082 systemd[1]: Stopping Ceph mon.smithi082 for 02181fb0-0d04-11ef-bc97-c7b262605968... 2024-05-08T06:27:59.262 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:27:59 smithi082 bash[22833]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon.smithi082" found: no such container 2024-05-08T06:27:59.263 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:27:59 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi082[22601]: 2024-05-08T06:27:59.181+0000 7f32988d9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi082 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-08T06:27:59.263 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:27:59 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi082[22601]: 2024-05-08T06:27:59.181+0000 7f32988d9700 -1 mon.smithi082@0(leader) e1 *** Got Signal Terminated *** 2024-05-08T06:27:59.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:27:59 smithi082 podman[22843]: 2024-05-08 06:27:59.263527132 +0000 UTC m=+0.194828316 container died 83f907e4f3895c7b55116b34a1a456294a071e9339e849f9ebfba702912f2f5a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi082, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, ceph=True, distribution-scope=public, name=centos-stream, release=754, GIT_CLEAN=True, io.openshift.expose-services=, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, vcs-type=git, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8) 2024-05-08T06:28:00.089 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:27:59 smithi082 podman[22843]: 2024-05-08 06:27:59.905979614 +0000 UTC m=+0.837280796 container cleanup 83f907e4f3895c7b55116b34a1a456294a071e9339e849f9ebfba702912f2f5a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi082, release=754, vcs-type=git, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, RELEASE=HEAD) 2024-05-08T06:28:00.089 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:27:59 smithi082 bash[22843]: ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi082 2024-05-08T06:28:00.359 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 podman[22856]: 2024-05-08 06:28:00.090328462 +0000 UTC m=+0.827600664 container remove 83f907e4f3895c7b55116b34a1a456294a071e9339e849f9ebfba702912f2f5a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi082, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, release=754, name=centos-stream, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, version=8, RELEASE=HEAD, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., GIT_CLEAN=True, architecture=x86_64, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-08T06:28:00.359 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 bash[22870]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon.smithi082" found: no such container 2024-05-08T06:28:00.360 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 systemd[1]: ceph-02181fb0-0d04-11ef-bc97-c7b262605968@mon.smithi082.service: Deactivated successfully. 2024-05-08T06:28:00.360 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 systemd[1]: Stopped Ceph mon.smithi082 for 02181fb0-0d04-11ef-bc97-c7b262605968. 2024-05-08T06:28:00.360 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 systemd[1]: Starting Ceph mon.smithi082 for 02181fb0-0d04-11ef-bc97-c7b262605968... 2024-05-08T06:28:00.634 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 podman[22938]: 2024-05-08 06:28:00.264620335 +0000 UTC m=+0.018801083 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-05-08T06:28:00.634 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 podman[22938]: 2024-05-08 06:28:00.366038904 +0000 UTC m=+0.120219677 container create ee82d4cf1e0f1a57ca19a0926527d64c21ad44178699e3fd6802ba5508a22543 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi082, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, version=8, architecture=x86_64, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, GIT_CLEAN=True, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, RELEASE=HEAD, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-08T06:28:00.634 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 podman[22938]: 2024-05-08 06:28:00.568533992 +0000 UTC m=+0.322714731 container init ee82d4cf1e0f1a57ca19a0926527d64c21ad44178699e3fd6802ba5508a22543 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi082, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, name=centos-stream, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, version=8, architecture=x86_64, maintainer=Guillaume Abrioux , RELEASE=HEAD, distribution-scope=public, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream) 2024-05-08T06:28:00.634 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 podman[22938]: 2024-05-08 06:28:00.571721071 +0000 UTC m=+0.325901805 container start ee82d4cf1e0f1a57ca19a0926527d64c21ad44178699e3fd6802ba5508a22543 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi082, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, GIT_BRANCH=HEAD, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.buildah.version=1.19.8, name=centos-stream, maintainer=Guillaume Abrioux , vcs-type=git, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, version=8, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, ceph=True) 2024-05-08T06:28:00.634 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: set uid:gid to 167:167 (ceph:ceph) 2024-05-08T06:28:00.634 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-05-08T06:28:00.634 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: pidfile_write: ignore empty --pid-file 2024-05-08T06:28:00.634 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: load: jerasure load: lrc 2024-05-08T06:28:00.634 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: RocksDB version: 6.15.5 2024-05-08T06:28:00.634 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-05-08T06:28:00.635 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Compile date Apr 18 2022 2024-05-08T06:28:00.635 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: DB SUMMARY 2024-05-08T06:28:00.635 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: DB Session ID: 4GUB292W0N2SJFLD5KUG 2024-05-08T06:28:00.635 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: CURRENT file: CURRENT 2024-05-08T06:28:00.635 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: IDENTITY file: IDENTITY 2024-05-08T06:28:00.635 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-05-08T06:28:00.635 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi082/store.db dir, Total Num: 1, files: 000008.sst 2024-05-08T06:28:00.635 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi082/store.db: 000010.log size: 82715 ; 2024-05-08T06:28:00.635 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.error_if_exists: 0 2024-05-08T06:28:00.635 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.create_if_missing: 0 2024-05-08T06:28:00.635 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.paranoid_checks: 1 2024-05-08T06:28:00.635 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-08T06:28:00.635 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.env: 0x55f505ac6860 2024-05-08T06:28:00.636 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.fs: Posix File System 2024-05-08T06:28:00.636 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.info_log: 0x55f5064edfa0 2024-05-08T06:28:00.636 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_file_opening_threads: 16 2024-05-08T06:28:00.636 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.statistics: (nil) 2024-05-08T06:28:00.636 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.use_fsync: 0 2024-05-08T06:28:00.636 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_log_file_size: 0 2024-05-08T06:28:00.636 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-08T06:28:00.637 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-08T06:28:00.637 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.keep_log_file_num: 1000 2024-05-08T06:28:00.637 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.recycle_log_file_num: 0 2024-05-08T06:28:00.638 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.allow_fallocate: 1 2024-05-08T06:28:00.638 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.allow_mmap_reads: 0 2024-05-08T06:28:00.638 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.allow_mmap_writes: 0 2024-05-08T06:28:00.638 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.use_direct_reads: 0 2024-05-08T06:28:00.638 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-08T06:28:00.638 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.create_missing_column_families: 0 2024-05-08T06:28:00.638 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.db_log_dir: 2024-05-08T06:28:00.638 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi082/store.db 2024-05-08T06:28:00.638 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-08T06:28:00.638 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-08T06:28:00.638 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-08T06:28:00.638 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-08T06:28:00.638 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-08T06:28:00.638 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-08T06:28:00.638 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.advise_random_on_open: 1 2024-05-08T06:28:00.639 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.db_write_buffer_size: 0 2024-05-08T06:28:00.639 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.write_buffer_manager: 0x55f5065de330 2024-05-08T06:28:00.639 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-08T06:28:00.639 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-05-08T06:28:00.639 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-08T06:28:00.639 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-08T06:28:00.639 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.rate_limiter: (nil) 2024-05-08T06:28:00.639 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-08T06:28:00.639 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.wal_recovery_mode: 2 2024-05-08T06:28:00.639 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.enable_thread_tracking: 0 2024-05-08T06:28:00.639 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.enable_pipelined_write: 0 2024-05-08T06:28:00.639 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.unordered_write: 0 2024-05-08T06:28:00.639 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-08T06:28:00.639 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-08T06:28:00.639 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-08T06:28:00.640 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-08T06:28:00.640 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.row_cache: None 2024-05-08T06:28:00.640 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.wal_filter: None 2024-05-08T06:28:00.640 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-08T06:28:00.640 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.allow_ingest_behind: 0 2024-05-08T06:28:00.640 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.preserve_deletes: 0 2024-05-08T06:28:00.640 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.two_write_queues: 0 2024-05-08T06:28:00.640 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.manual_wal_flush: 0 2024-05-08T06:28:00.640 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.atomic_flush: 0 2024-05-08T06:28:00.640 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-08T06:28:00.640 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-08T06:28:00.640 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-08T06:28:00.640 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.log_readahead_size: 0 2024-05-08T06:28:00.640 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-08T06:28:00.640 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.best_efforts_recovery: 0 2024-05-08T06:28:00.641 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-08T06:28:00.641 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-08T06:28:00.641 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.allow_data_in_errors: 0 2024-05-08T06:28:00.641 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.db_host_id: __hostname__ 2024-05-08T06:28:00.641 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_background_jobs: 2 2024-05-08T06:28:00.641 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_background_compactions: -1 2024-05-08T06:28:00.641 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_subcompactions: 1 2024-05-08T06:28:00.641 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-08T06:28:00.641 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-08T06:28:00.641 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-08T06:28:00.641 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_total_wal_size: 0 2024-05-08T06:28:00.641 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-08T06:28:00.641 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-08T06:28:00.641 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-08T06:28:00.642 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-08T06:28:00.642 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_open_files: -1 2024-05-08T06:28:00.642 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.bytes_per_sync: 0 2024-05-08T06:28:00.642 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-08T06:28:00.642 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-08T06:28:00.642 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compaction_readahead_size: 0 2024-05-08T06:28:00.642 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_background_flushes: -1 2024-05-08T06:28:00.642 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Compression algorithms supported: 2024-05-08T06:28:00.642 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-08T06:28:00.642 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: kZSTD supported: 0 2024-05-08T06:28:00.642 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: kXpressCompression supported: 0 2024-05-08T06:28:00.642 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: kLZ4HCCompression supported: 1 2024-05-08T06:28:00.642 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: kLZ4Compression supported: 1 2024-05-08T06:28:00.642 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: kBZip2Compression supported: 0 2024-05-08T06:28:00.643 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: kZlibCompression supported: 1 2024-05-08T06:28:00.643 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: kSnappyCompression supported: 1 2024-05-08T06:28:00.643 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-08T06:28:00.643 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi082/store.db/MANIFEST-000009 2024-05-08T06:28:00.643 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-05-08T06:28:00.643 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-08T06:28:00.643 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.merge_operator: 2024-05-08T06:28:00.643 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compaction_filter: None 2024-05-08T06:28:00.643 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compaction_filter_factory: None 2024-05-08T06:28:00.643 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.sst_partitioner_factory: None 2024-05-08T06:28:00.643 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-08T06:28:00.643 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-08T06:28:00.643 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f5064b9d00) 2024-05-08T06:28:00.643 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: cache_index_and_filter_blocks: 1 2024-05-08T06:28:00.643 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-08T06:28:00.644 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-08T06:28:00.644 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: pin_top_level_index_and_filter: 1 2024-05-08T06:28:00.644 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: index_type: 0 2024-05-08T06:28:00.644 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: data_block_index_type: 0 2024-05-08T06:28:00.644 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: index_shortening: 1 2024-05-08T06:28:00.644 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-08T06:28:00.644 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: hash_index_allow_collision: 1 2024-05-08T06:28:00.644 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: checksum: 1 2024-05-08T06:28:00.644 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: no_block_cache: 0 2024-05-08T06:28:00.644 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: block_cache: 0x55f506524170 2024-05-08T06:28:00.644 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: block_cache_name: BinnedLRUCache 2024-05-08T06:28:00.644 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: block_cache_options: 2024-05-08T06:28:00.644 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: capacity : 536870912 2024-05-08T06:28:00.644 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: num_shard_bits : 4 2024-05-08T06:28:00.644 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: strict_capacity_limit : 0 2024-05-08T06:28:00.645 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: high_pri_pool_ratio: 0.000 2024-05-08T06:28:00.645 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: block_cache_compressed: (nil) 2024-05-08T06:28:00.645 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: persistent_cache: (nil) 2024-05-08T06:28:00.645 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: block_size: 4096 2024-05-08T06:28:00.645 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: block_size_deviation: 10 2024-05-08T06:28:00.645 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: block_restart_interval: 16 2024-05-08T06:28:00.645 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: index_block_restart_interval: 1 2024-05-08T06:28:00.645 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: metadata_block_size: 4096 2024-05-08T06:28:00.645 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: partition_filters: 0 2024-05-08T06:28:00.645 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: use_delta_encoding: 1 2024-05-08T06:28:00.645 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-05-08T06:28:00.645 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: whole_key_filtering: 1 2024-05-08T06:28:00.645 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: verify_compression: 0 2024-05-08T06:28:00.645 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: read_amp_bytes_per_bit: 0 2024-05-08T06:28:00.646 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: format_version: 4 2024-05-08T06:28:00.646 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: enable_index_compression: 1 2024-05-08T06:28:00.646 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: block_align: 0 2024-05-08T06:28:00.646 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.write_buffer_size: 33554432 2024-05-08T06:28:00.646 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_write_buffer_number: 2 2024-05-08T06:28:00.646 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compression: NoCompression 2024-05-08T06:28:00.646 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.bottommost_compression: Disabled 2024-05-08T06:28:00.646 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.prefix_extractor: nullptr 2024-05-08T06:28:00.646 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-08T06:28:00.646 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.num_levels: 7 2024-05-08T06:28:00.646 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-08T06:28:00.646 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-08T06:28:00.646 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-08T06:28:00.646 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-08T06:28:00.646 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-08T06:28:00.647 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-08T06:28:00.647 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-08T06:28:00.647 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-08T06:28:00.647 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-08T06:28:00.647 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-08T06:28:00.647 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-08T06:28:00.647 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compression_opts.level: 32767 2024-05-08T06:28:00.647 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compression_opts.strategy: 0 2024-05-08T06:28:00.647 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-08T06:28:00.647 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-08T06:28:00.647 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-08T06:28:00.647 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compression_opts.enabled: false 2024-05-08T06:28:00.647 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-08T06:28:00.647 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-08T06:28:00.647 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-08T06:28:00.648 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.target_file_size_base: 67108864 2024-05-08T06:28:00.648 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-08T06:28:00.648 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-08T06:28:00.648 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-08T06:28:00.648 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-08T06:28:00.648 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-08T06:28:00.648 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-08T06:28:00.648 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-08T06:28:00.648 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-08T06:28:00.648 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-08T06:28:00.648 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-08T06:28:00.648 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-08T06:28:00.648 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-08T06:28:00.648 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-08T06:28:00.648 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.arena_block_size: 4194304 2024-05-08T06:28:00.649 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-08T06:28:00.649 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-08T06:28:00.649 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-05-08T06:28:00.649 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.disable_auto_compactions: 0 2024-05-08T06:28:00.649 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-08T06:28:00.649 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-08T06:28:00.649 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-08T06:28:00.649 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-08T06:28:00.649 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-08T06:28:00.649 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-08T06:28:00.649 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-08T06:28:00.649 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-08T06:28:00.649 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-08T06:28:00.649 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-08T06:28:00.650 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.table_properties_collectors: 2024-05-08T06:28:00.650 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.inplace_update_support: 0 2024-05-08T06:28:00.650 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-08T06:28:00.650 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-08T06:28:00.650 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-08T06:28:00.650 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-08T06:28:00.650 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.bloom_locality: 0 2024-05-08T06:28:00.650 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.max_successive_merges: 0 2024-05-08T06:28:00.650 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-08T06:28:00.650 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.paranoid_file_checks: 0 2024-05-08T06:28:00.650 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.force_consistency_checks: 1 2024-05-08T06:28:00.650 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.report_bg_io_stats: 0 2024-05-08T06:28:00.650 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.ttl: 2592000 2024-05-08T06:28:00.650 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-08T06:28:00.650 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.enable_blob_files: false 2024-05-08T06:28:00.651 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.min_blob_size: 0 2024-05-08T06:28:00.651 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.blob_file_size: 268435456 2024-05-08T06:28:00.651 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-08T06:28:00.651 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-08T06:28:00.651 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-08T06:28:00.651 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi082/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-08T06:28:00.651 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-05-08T06:28:00.651 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-05-08T06:28:00.651 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715149680597724, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-05-08T06:28:00.651 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-05-08T06:28:00.651 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-05-08T06:28:00.651 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715149680598514, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79397, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77661, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71970, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1715149680, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "16561780-e051-4f5d-b1b0-2b080c4fe558", "db_session_id": "4GUB292W0N2SJFLD5KUG"}} 2024-05-08T06:28:00.651 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-05-08T06:28:00.651 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: EVENT_LOG_v1 {"time_micros": 1715149680601609, "job": 1, "event": "recovery_finished"} 2024-05-08T06:28:00.651 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi082/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-08T06:28:00.652 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55f50650aa80 2024-05-08T06:28:00.652 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: DB pointer 0x55f50651a000 2024-05-08T06:28:00.652 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-05-08T06:28:00.652 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-05-08T06:28:00.652 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: ** DB Stats ** 2024-05-08T06:28:00.652 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-08T06:28:00.652 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-08T06:28:00.652 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-08T06:28:00.652 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-08T06:28:00.652 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-08T06:28:00.652 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-05-08T06:28:00.652 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-08T06:28:00.652 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: 2024-05-08T06:28:00.652 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: ** Compaction Stats [default] ** 2024-05-08T06:28:00.652 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-08T06:28:00.653 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-08T06:28:00.653 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.4 0.00 0.00 1 0.001 0 0 2024-05-08T06:28:00.653 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.4 0.00 0.00 1 0.001 0 0 2024-05-08T06:28:00.653 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.4 0.00 0.00 1 0.001 0 0 2024-05-08T06:28:00.653 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: 2024-05-08T06:28:00.653 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: ** Compaction Stats [default] ** 2024-05-08T06:28:00.653 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-08T06:28:00.653 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-08T06:28:00.653 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 137.4 0.00 0.00 1 0.001 0 0 2024-05-08T06:28:00.653 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-08T06:28:00.653 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-08T06:28:00.653 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-08T06:28:00.653 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-08T06:28:00.653 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-08T06:28:00.653 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-08T06:28:00.654 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Cumulative compaction: 0.00 GB write, 12.83 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-08T06:28:00.654 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Interval compaction: 0.00 GB write, 12.83 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-08T06:28:00.654 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-08T06:28:00.654 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: 2024-05-08T06:28:00.654 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-08T06:28:00.654 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: 2024-05-08T06:28:00.654 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: ** Compaction Stats [default] ** 2024-05-08T06:28:00.654 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-08T06:28:00.654 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-08T06:28:00.654 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.4 0.00 0.00 1 0.001 0 0 2024-05-08T06:28:00.654 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.4 0.00 0.00 1 0.001 0 0 2024-05-08T06:28:00.654 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-05-08T06:28:00.654 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: 2024-05-08T06:28:00.654 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: ** Compaction Stats [default] ** 2024-05-08T06:28:00.654 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-08T06:28:00.655 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-08T06:28:00.655 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 137.4 0.00 0.00 1 0.001 0 0 2024-05-08T06:28:00.655 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-08T06:28:00.655 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-08T06:28:00.655 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-08T06:28:00.655 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-08T06:28:00.655 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-08T06:28:00.655 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-08T06:28:00.655 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Cumulative compaction: 0.00 GB write, 12.76 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-08T06:28:00.655 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-08T06:28:00.655 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-08T06:28:00.655 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: 2024-05-08T06:28:00.655 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-08T06:28:00.655 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: starting mon.smithi082 rank 0 at public addrs [v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0] at bind addrs [v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi082 fsid 02181fb0-0d04-11ef-bc97-c7b262605968 2024-05-08T06:28:00.655 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: mon.smithi082@-1(???) e1 preinit fsid 02181fb0-0d04-11ef-bc97-c7b262605968 2024-05-08T06:28:00.656 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: mon.smithi082@-1(???).mds e1 new map 2024-05-08T06:28:00.656 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: mon.smithi082@-1(???).mds e1 print_map 2024-05-08T06:28:00.656 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: e1 2024-05-08T06:28:00.656 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-08T06:28:00.656 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-08T06:28:00.656 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: legacy client fscid: -1 2024-05-08T06:28:00.656 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: 2024-05-08T06:28:00.656 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout: No filesystems configured 2024-05-08T06:28:00.656 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: mon.smithi082@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-05-08T06:28:00.656 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: mon.smithi082@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-08T06:28:00.656 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: mon.smithi082@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-08T06:28:00.656 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: mon.smithi082@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-08T06:28:00.656 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: mon.smithi082@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-05-08T06:28:00.656 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 bash[22938]: ee82d4cf1e0f1a57ca19a0926527d64c21ad44178699e3fd6802ba5508a22543 2024-05-08T06:28:00.656 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 systemd[1]: Started Ceph mon.smithi082 for 02181fb0-0d04-11ef-bc97-c7b262605968. 2024-05-08T06:28:00.657 INFO:teuthology.orchestra.run.smithi082.stderr:Setting mon public_network to 172.21.0.0/20 2024-05-08T06:28:00.912 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: mon.smithi082 is new leader, mons smithi082 in quorum (ranks 0) 2024-05-08T06:28:00.913 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: monmap e1: 1 mons at {smithi082=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0]} 2024-05-08T06:28:00.913 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: fsmap 2024-05-08T06:28:00.913 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: osdmap e1: 0 total, 0 up, 0 in 2024-05-08T06:28:00.913 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:00 smithi082 ceph-mon[22957]: mgrmap e1: no daemons active 2024-05-08T06:28:01.593 INFO:teuthology.orchestra.run.smithi082.stderr:Wrote config to /etc/ceph/ceph.conf 2024-05-08T06:28:01.593 INFO:teuthology.orchestra.run.smithi082.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-05-08T06:28:01.593 INFO:teuthology.orchestra.run.smithi082.stderr:Creating mgr... 2024-05-08T06:28:01.593 INFO:teuthology.orchestra.run.smithi082.stderr:Verifying port 9283 ... 2024-05-08T06:28:01.767 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Failed to reset failed state of unit ceph-02181fb0-0d04-11ef-bc97-c7b262605968@mgr.smithi082.bsnvtb.service: Unit ceph-02181fb0-0d04-11ef-bc97-c7b262605968@mgr.smithi082.bsnvtb.service not loaded. 2024-05-08T06:28:01.774 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: Created symlink /etc/systemd/system/ceph-02181fb0-0d04-11ef-bc97-c7b262605968.target.wants/ceph-02181fb0-0d04-11ef-bc97-c7b262605968@mgr.smithi082.bsnvtb.service → /etc/systemd/system/ceph-02181fb0-0d04-11ef-bc97-c7b262605968@.service. 2024-05-08T06:28:02.359 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:02 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/1317246900' entity='client.admin' 2024-05-08T06:28:02.472 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: disabled 2024-05-08T06:28:02.480 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: inactive 2024-05-08T06:28:02.480 INFO:teuthology.orchestra.run.smithi082.stderr:firewalld.service is not enabled 2024-05-08T06:28:02.480 INFO:teuthology.orchestra.run.smithi082.stderr:Not possible to enable service . firewalld.service is not available 2024-05-08T06:28:02.489 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: disabled 2024-05-08T06:28:02.496 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: inactive 2024-05-08T06:28:02.497 INFO:teuthology.orchestra.run.smithi082.stderr:firewalld.service is not enabled 2024-05-08T06:28:02.497 INFO:teuthology.orchestra.run.smithi082.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-05-08T06:28:02.497 INFO:teuthology.orchestra.run.smithi082.stderr:Waiting for mgr to start... 2024-05-08T06:28:02.497 INFO:teuthology.orchestra.run.smithi082.stderr:Waiting for mgr... 2024-05-08T06:28:03.037 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2024-05-08T06:28:03.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: { 2024-05-08T06:28:03.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "fsid": "02181fb0-0d04-11ef-bc97-c7b262605968", 2024-05-08T06:28:03.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "health": { 2024-05-08T06:28:03.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-08T06:28:03.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "checks": {}, 2024-05-08T06:28:03.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "mutes": [] 2024-05-08T06:28:03.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:03.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-08T06:28:03.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "quorum": [ 2024-05-08T06:28:03.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 0 2024-05-08T06:28:03.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ], 2024-05-08T06:28:03.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-08T06:28:03.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "smithi082" 2024-05-08T06:28:03.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ], 2024-05-08T06:28:03.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-05-08T06:28:03.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "monmap": { 2024-05-08T06:28:03.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-08T06:28:03.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-08T06:28:03.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-08T06:28:03.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:03.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "osdmap": { 2024-05-08T06:28:03.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-08T06:28:03.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-08T06:28:03.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-08T06:28:03.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-08T06:28:03.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-08T06:28:03.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-08T06:28:03.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-08T06:28:03.039 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:03.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "pgmap": { 2024-05-08T06:28:03.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-08T06:28:03.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-08T06:28:03.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-08T06:28:03.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-08T06:28:03.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-08T06:28:03.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-08T06:28:03.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-08T06:28:03.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-08T06:28:03.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:03.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "fsmap": { 2024-05-08T06:28:03.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-08T06:28:03.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-08T06:28:03.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-08T06:28:03.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:03.040 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-08T06:28:03.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "available": false, 2024-05-08T06:28:03.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-08T06:28:03.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "modules": [ 2024-05-08T06:28:03.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "iostat", 2024-05-08T06:28:03.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "nfs", 2024-05-08T06:28:03.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "restful" 2024-05-08T06:28:03.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ], 2024-05-08T06:28:03.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "services": {} 2024-05-08T06:28:03.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:03.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "servicemap": { 2024-05-08T06:28:03.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-08T06:28:03.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "modified": "2024-05-08T06:27:55.743370+0000", 2024-05-08T06:28:03.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "services": {} 2024-05-08T06:28:03.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:03.041 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-08T06:28:03.042 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: } 2024-05-08T06:28:03.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:03 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/2769729072' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-08T06:28:04.319 INFO:teuthology.orchestra.run.smithi082.stderr:mgr not available, waiting (1/15)... 2024-05-08T06:28:06.859 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2024-05-08T06:28:06.859 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: { 2024-05-08T06:28:06.859 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "fsid": "02181fb0-0d04-11ef-bc97-c7b262605968", 2024-05-08T06:28:06.860 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "health": { 2024-05-08T06:28:06.860 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-08T06:28:06.860 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "checks": {}, 2024-05-08T06:28:06.860 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "mutes": [] 2024-05-08T06:28:06.860 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:06.860 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-08T06:28:06.860 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "quorum": [ 2024-05-08T06:28:06.860 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 0 2024-05-08T06:28:06.860 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ], 2024-05-08T06:28:06.860 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-08T06:28:06.860 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "smithi082" 2024-05-08T06:28:06.860 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ], 2024-05-08T06:28:06.860 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-05-08T06:28:06.860 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "monmap": { 2024-05-08T06:28:06.860 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-08T06:28:06.860 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-08T06:28:06.861 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-08T06:28:06.861 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:06.861 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "osdmap": { 2024-05-08T06:28:06.861 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-08T06:28:06.861 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-08T06:28:06.861 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-08T06:28:06.861 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-08T06:28:06.861 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-08T06:28:06.864 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-08T06:28:06.864 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-08T06:28:06.865 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:06.865 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "pgmap": { 2024-05-08T06:28:06.865 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-08T06:28:06.865 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-08T06:28:06.865 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-08T06:28:06.865 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-08T06:28:06.865 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-08T06:28:06.865 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-08T06:28:06.865 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-08T06:28:06.865 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-08T06:28:06.865 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:06.865 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "fsmap": { 2024-05-08T06:28:06.865 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-08T06:28:06.866 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-08T06:28:06.866 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-08T06:28:06.866 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:06.866 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-08T06:28:06.866 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "available": false, 2024-05-08T06:28:06.866 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-08T06:28:06.866 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "modules": [ 2024-05-08T06:28:06.866 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "iostat", 2024-05-08T06:28:06.866 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "nfs", 2024-05-08T06:28:06.866 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "restful" 2024-05-08T06:28:06.866 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ], 2024-05-08T06:28:06.866 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "services": {} 2024-05-08T06:28:06.866 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:06.867 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "servicemap": { 2024-05-08T06:28:06.867 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-08T06:28:06.867 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "modified": "2024-05-08T06:27:55.743370+0000", 2024-05-08T06:28:06.867 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "services": {} 2024-05-08T06:28:06.867 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:06.867 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-08T06:28:06.867 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: } 2024-05-08T06:28:07.155 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:06 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/1690623837' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-08T06:28:07.474 INFO:teuthology.orchestra.run.smithi082.stderr:mgr not available, waiting (2/15)... 2024-05-08T06:28:08.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:08 smithi082 ceph-mon[22957]: Activating manager daemon smithi082.bsnvtb 2024-05-08T06:28:08.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:08 smithi082 ceph-mon[22957]: mgrmap e2: smithi082.bsnvtb(active, starting, since 0.00277702s) 2024-05-08T06:28:08.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:08 smithi082 ceph-mon[22957]: from='mgr.14100 172.21.15.82:0/2614998493' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-08T06:28:08.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:08 smithi082 ceph-mon[22957]: from='mgr.14100 172.21.15.82:0/2614998493' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-08T06:28:08.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:08 smithi082 ceph-mon[22957]: from='mgr.14100 172.21.15.82:0/2614998493' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-08T06:28:08.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:08 smithi082 ceph-mon[22957]: from='mgr.14100 172.21.15.82:0/2614998493' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi082"}]: dispatch 2024-05-08T06:28:08.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:08 smithi082 ceph-mon[22957]: from='mgr.14100 172.21.15.82:0/2614998493' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mgr metadata", "who": "smithi082.bsnvtb", "id": "smithi082.bsnvtb"}]: dispatch 2024-05-08T06:28:08.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:08 smithi082 ceph-mon[22957]: Manager daemon smithi082.bsnvtb is now available 2024-05-08T06:28:08.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:08 smithi082 ceph-mon[22957]: from='mgr.14100 172.21.15.82:0/2614998493' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:28:08.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:08 smithi082 ceph-mon[22957]: from='mgr.14100 172.21.15.82:0/2614998493' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:28:08.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:08 smithi082 ceph-mon[22957]: from='mgr.14100 172.21.15.82:0/2614998493' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:08.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:08 smithi082 ceph-mon[22957]: from='mgr.14100 172.21.15.82:0/2614998493' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:08.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:08 smithi082 ceph-mon[22957]: from='mgr.14100 172.21.15.82:0/2614998493' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:10.153 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2024-05-08T06:28:10.153 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: { 2024-05-08T06:28:10.153 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "fsid": "02181fb0-0d04-11ef-bc97-c7b262605968", 2024-05-08T06:28:10.154 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "health": { 2024-05-08T06:28:10.154 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-08T06:28:10.154 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "checks": {}, 2024-05-08T06:28:10.154 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "mutes": [] 2024-05-08T06:28:10.154 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:10.154 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-08T06:28:10.154 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "quorum": [ 2024-05-08T06:28:10.154 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 0 2024-05-08T06:28:10.154 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ], 2024-05-08T06:28:10.154 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-08T06:28:10.154 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "smithi082" 2024-05-08T06:28:10.154 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ], 2024-05-08T06:28:10.154 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-05-08T06:28:10.154 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "monmap": { 2024-05-08T06:28:10.155 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-08T06:28:10.155 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-08T06:28:10.155 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-08T06:28:10.155 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:10.155 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "osdmap": { 2024-05-08T06:28:10.155 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-08T06:28:10.155 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-08T06:28:10.155 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-08T06:28:10.155 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-08T06:28:10.155 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-08T06:28:10.155 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-08T06:28:10.155 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-08T06:28:10.155 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:10.155 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "pgmap": { 2024-05-08T06:28:10.155 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-08T06:28:10.156 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-08T06:28:10.156 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-08T06:28:10.158 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-08T06:28:10.158 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-08T06:28:10.159 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-08T06:28:10.159 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-08T06:28:10.159 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-08T06:28:10.159 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:10.159 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "fsmap": { 2024-05-08T06:28:10.159 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-08T06:28:10.159 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-08T06:28:10.159 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-08T06:28:10.159 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:10.159 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-08T06:28:10.159 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "available": true, 2024-05-08T06:28:10.159 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-08T06:28:10.159 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "modules": [ 2024-05-08T06:28:10.159 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "iostat", 2024-05-08T06:28:10.160 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "nfs", 2024-05-08T06:28:10.160 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "restful" 2024-05-08T06:28:10.160 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ], 2024-05-08T06:28:10.160 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "services": {} 2024-05-08T06:28:10.160 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:10.160 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "servicemap": { 2024-05-08T06:28:10.160 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-08T06:28:10.160 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "modified": "2024-05-08T06:27:55.743370+0000", 2024-05-08T06:28:10.160 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "services": {} 2024-05-08T06:28:10.160 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: }, 2024-05-08T06:28:10.160 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-08T06:28:10.160 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: } 2024-05-08T06:28:10.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:10 smithi082 ceph-mon[22957]: mgrmap e3: smithi082.bsnvtb(active, since 1.00594s) 2024-05-08T06:28:10.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:10 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/243919678' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-08T06:28:10.730 INFO:teuthology.orchestra.run.smithi082.stderr:mgr is available 2024-05-08T06:28:11.299 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2024-05-08T06:28:11.299 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: [global] 2024-05-08T06:28:11.299 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: fsid = 02181fb0-0d04-11ef-bc97-c7b262605968 2024-05-08T06:28:11.299 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-05-08T06:28:11.299 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-05-08T06:28:11.299 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-05-08T06:28:11.299 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-05-08T06:28:11.299 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2024-05-08T06:28:11.299 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: [mgr] 2024-05-08T06:28:11.299 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-05-08T06:28:11.299 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 2024-05-08T06:28:11.299 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: [osd] 2024-05-08T06:28:11.299 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-05-08T06:28:11.300 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-05-08T06:28:11.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:11 smithi082 ceph-mon[22957]: mgrmap e4: smithi082.bsnvtb(active, since 2s) 2024-05-08T06:28:11.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:11 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3594663329' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-05-08T06:28:12.004 INFO:teuthology.orchestra.run.smithi082.stderr:Enabling cephadm module... 2024-05-08T06:28:13.032 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:12 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3388271367' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-05-08T06:28:13.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:13 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3388271367' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-05-08T06:28:13.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:13 smithi082 ceph-mon[22957]: mgrmap e5: smithi082.bsnvtb(active, since 4s) 2024-05-08T06:28:14.037 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: { 2024-05-08T06:28:14.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 5, 2024-05-08T06:28:14.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "available": true, 2024-05-08T06:28:14.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "active_name": "smithi082.bsnvtb", 2024-05-08T06:28:14.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_standby": 0 2024-05-08T06:28:14.038 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: } 2024-05-08T06:28:14.616 INFO:teuthology.orchestra.run.smithi082.stderr:Waiting for the mgr to restart... 2024-05-08T06:28:14.616 INFO:teuthology.orchestra.run.smithi082.stderr:Waiting for mgr epoch 5... 2024-05-08T06:28:14.887 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:14 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/2175495780' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-05-08T06:28:19.864 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: { 2024-05-08T06:28:19.864 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-05-08T06:28:19.864 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "initialized": true 2024-05-08T06:28:19.864 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: } 2024-05-08T06:28:20.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:19 smithi082 ceph-mon[22957]: Active manager daemon smithi082.bsnvtb restarted 2024-05-08T06:28:20.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:19 smithi082 ceph-mon[22957]: Activating manager daemon smithi082.bsnvtb 2024-05-08T06:28:20.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:19 smithi082 ceph-mon[22957]: osdmap e2: 0 total, 0 up, 0 in 2024-05-08T06:28:20.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:19 smithi082 ceph-mon[22957]: mgrmap e6: smithi082.bsnvtb(active, starting, since 0.0525642s) 2024-05-08T06:28:20.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:19 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi082"}]: dispatch 2024-05-08T06:28:20.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:19 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mgr metadata", "who": "smithi082.bsnvtb", "id": "smithi082.bsnvtb"}]: dispatch 2024-05-08T06:28:20.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:19 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-08T06:28:20.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:19 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-08T06:28:20.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:19 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-08T06:28:20.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:19 smithi082 ceph-mon[22957]: Manager daemon smithi082.bsnvtb is now available 2024-05-08T06:28:20.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:19 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:20.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:19 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:20.169 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:19 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:28:20.169 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:19 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:28:20.169 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:19 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:28:20.169 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:19 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:28:20.169 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:19 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:28:20.372 INFO:teuthology.orchestra.run.smithi082.stderr:mgr epoch 5 is available 2024-05-08T06:28:20.372 INFO:teuthology.orchestra.run.smithi082.stderr:Setting orchestrator backend to cephadm... 2024-05-08T06:28:21.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:20 smithi082 ceph-mon[22957]: mgrmap e7: smithi082.bsnvtb(active, since 1.05452s) 2024-05-08T06:28:21.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:20 smithi082 ceph-mon[22957]: [08/May/2024:06:28:20] ENGINE Bus STARTING 2024-05-08T06:28:21.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:20 smithi082 ceph-mon[22957]: [08/May/2024:06:28:20] ENGINE Serving on https://172.21.15.82:7150 2024-05-08T06:28:21.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:20 smithi082 ceph-mon[22957]: [08/May/2024:06:28:20] ENGINE Bus STARTED 2024-05-08T06:28:21.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:20 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:21.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:20 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:28:22.173 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: value unchanged 2024-05-08T06:28:22.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:21 smithi082 ceph-mon[22957]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:28:22.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:21 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:22.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:21 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:28:22.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:21 smithi082 ceph-mon[22957]: mgrmap e8: smithi082.bsnvtb(active, since 2s) 2024-05-08T06:28:22.743 INFO:teuthology.orchestra.run.smithi082.stderr:Generating ssh key... 2024-05-08T06:28:22.993 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:22 smithi082 ceph-mon[22957]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:28:24.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:24 smithi082 ceph-mon[22957]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:28:24.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:24 smithi082 ceph-mon[22957]: Generating ssh key... 2024-05-08T06:28:24.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:24 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:24.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:24 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:24.757 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDYy65sFsD2B3dk2wCxff+RMU4K2nh8ABaf3mtX9vCnWduV/xQeFqqUodU+IQ24QfRLnJyscTko4er0VYrA26Iuelyj3wjE8+9qWiNX7EqCPgMHl8IZpHv2Jb/2hAF3yTQxEny0r9RMZkbGishoER4NpU1E3eR73zCs1d14Zg4pU68BBL8EaYSAZTwtLDOlKDRAPPXTjI/0qi0aOYkT1mL631hNrnBv14IpuHY8isFPj5rZeVIvuE32iALgu87dr6HzlWUtzErV2FVS7EPXH781n+LqJ0HlbtGVORuQXFnHWxT1poq8dMMDNchdWbbbIiq4J7y/T+Bszcms9ANSDUraAWp1smadukx/jbozxQONPqK0zCDk7iTgXNh5vB36Cnlg6NsspP4CS/2rb7SSR9nyy00ufcSPqPL3sxKfvU37xOCyPraxhLIvetplSqChoDSfbqa+n5kOGeu/gVdvHcVdoJsO5XsrqKpjByAQYaRlOatNj0Oas3UWxaYckH8DQOU= ceph-02181fb0-0d04-11ef-bc97-c7b262605968 2024-05-08T06:28:25.298 INFO:teuthology.orchestra.run.smithi082.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-05-08T06:28:25.299 INFO:teuthology.orchestra.run.smithi082.stderr:Adding key to root@localhost authorized_keys... 2024-05-08T06:28:25.299 INFO:teuthology.orchestra.run.smithi082.stderr:Adding host smithi082... 2024-05-08T06:28:25.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:25 smithi082 ceph-mon[22957]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:28:26.779 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: Added host 'smithi082' with addr '172.21.15.82' 2024-05-08T06:28:27.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:26 smithi082 ceph-mon[22957]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi082", "addr": "172.21.15.82", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:28:27.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:26 smithi082 ceph-mon[22957]: Deploying cephadm binary to smithi082 2024-05-08T06:28:27.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:26 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:27.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:26 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:28:27.305 INFO:teuthology.orchestra.run.smithi082.stderr:Deploying mon service with default placement... 2024-05-08T06:28:27.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:27 smithi082 ceph-mon[22957]: Added host smithi082 2024-05-08T06:28:27.923 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: Scheduled mon update... 2024-05-08T06:28:28.689 INFO:teuthology.orchestra.run.smithi082.stderr:Deploying mgr service with default placement... 2024-05-08T06:28:29.027 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:28 smithi082 ceph-mon[22957]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:28:29.027 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:28 smithi082 ceph-mon[22957]: Saving service mon spec with placement count:5 2024-05-08T06:28:29.027 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:28 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:29.450 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-05-08T06:28:30.859 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:30 smithi082 ceph-mon[22957]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:28:30.859 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:30 smithi082 ceph-mon[22957]: Saving service mgr spec with placement count:2 2024-05-08T06:28:30.859 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:30 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:30.861 INFO:teuthology.orchestra.run.smithi082.stderr:Deploying crash service with default placement... 2024-05-08T06:28:31.465 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: Scheduled crash update... 2024-05-08T06:28:31.719 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:31 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:31.719 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:31 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:31.980 INFO:teuthology.orchestra.run.smithi082.stderr:Deploying prometheus service with default placement... 2024-05-08T06:28:32.601 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:32 smithi082 ceph-mon[22957]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:28:32.601 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:32 smithi082 ceph-mon[22957]: Saving service crash spec with placement * 2024-05-08T06:28:32.601 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:32 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:32.601 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-05-08T06:28:33.693 INFO:teuthology.orchestra.run.smithi082.stderr:Deploying grafana service with default placement... 2024-05-08T06:28:33.815 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:33 smithi082 ceph-mon[22957]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:28:33.815 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:33 smithi082 ceph-mon[22957]: Saving service prometheus spec with placement count:1 2024-05-08T06:28:33.815 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:33 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:33.815 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:33 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:34.345 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-05-08T06:28:34.904 INFO:teuthology.orchestra.run.smithi082.stderr:Deploying node-exporter service with default placement... 2024-05-08T06:28:35.497 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:35 smithi082 ceph-mon[22957]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:28:35.497 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:35 smithi082 ceph-mon[22957]: Saving service grafana spec with placement count:1 2024-05-08T06:28:35.498 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:35 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:35.498 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:35 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:35.538 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-05-08T06:28:36.584 INFO:teuthology.orchestra.run.smithi082.stderr:Deploying alertmanager service with default placement... 2024-05-08T06:28:36.804 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:36 smithi082 ceph-mon[22957]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:28:36.804 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:36 smithi082 ceph-mon[22957]: Saving service node-exporter spec with placement * 2024-05-08T06:28:36.804 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:36 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:37.522 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-05-08T06:28:38.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:38 smithi082 ceph-mon[22957]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:28:38.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:38 smithi082 ceph-mon[22957]: Saving service alertmanager spec with placement count:1 2024-05-08T06:28:38.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:38 smithi082 ceph-mon[22957]: from='mgr.14118 172.21.15.82:0/2266353967' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:40.081 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:39 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/1317928183' entity='client.admin' 2024-05-08T06:28:40.081 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:39 smithi082 ceph-mon[22957]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:28:40.705 INFO:teuthology.orchestra.run.smithi082.stderr:Enabling the dashboard module... 2024-05-08T06:28:41.147 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:41 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3070988967' entity='client.admin' 2024-05-08T06:28:42.351 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:42 smithi082 ceph-mon[22957]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:28:42.351 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:42 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/2125624917' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-05-08T06:28:43.349 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:43 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/2125624917' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-05-08T06:28:43.349 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:43 smithi082 ceph-mon[22957]: mgrmap e9: smithi082.bsnvtb(active, since 23s) 2024-05-08T06:28:43.436 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: { 2024-05-08T06:28:43.437 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "epoch": 9, 2024-05-08T06:28:43.437 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "available": true, 2024-05-08T06:28:43.437 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "active_name": "smithi082.bsnvtb", 2024-05-08T06:28:43.437 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "num_standby": 0 2024-05-08T06:28:43.437 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: } 2024-05-08T06:28:44.150 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:44 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3060896377' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-05-08T06:28:44.153 INFO:teuthology.orchestra.run.smithi082.stderr:Waiting for the mgr to restart... 2024-05-08T06:28:44.153 INFO:teuthology.orchestra.run.smithi082.stderr:Waiting for mgr epoch 9... 2024-05-08T06:28:49.255 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: { 2024-05-08T06:28:49.255 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-05-08T06:28:49.255 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: "initialized": true 2024-05-08T06:28:49.255 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: } 2024-05-08T06:28:49.595 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:49 smithi082 ceph-mon[22957]: Active manager daemon smithi082.bsnvtb restarted 2024-05-08T06:28:49.595 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:49 smithi082 ceph-mon[22957]: Activating manager daemon smithi082.bsnvtb 2024-05-08T06:28:49.595 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:49 smithi082 ceph-mon[22957]: osdmap e3: 0 total, 0 up, 0 in 2024-05-08T06:28:49.595 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:49 smithi082 ceph-mon[22957]: mgrmap e10: smithi082.bsnvtb(active, starting, since 0.0551194s) 2024-05-08T06:28:49.595 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:49 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi082"}]: dispatch 2024-05-08T06:28:49.595 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:49 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mgr metadata", "who": "smithi082.bsnvtb", "id": "smithi082.bsnvtb"}]: dispatch 2024-05-08T06:28:49.595 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:49 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-08T06:28:49.596 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:49 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-08T06:28:49.596 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:49 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-08T06:28:49.596 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:49 smithi082 ceph-mon[22957]: Manager daemon smithi082.bsnvtb is now available 2024-05-08T06:28:49.596 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:49 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:28:49.596 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:49 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:28:49.596 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:49 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:28:49.596 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:49 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:28:49.596 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:49 smithi082 ceph-mon[22957]: [08/May/2024:06:28:49] ENGINE Bus STARTING 2024-05-08T06:28:49.596 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:49 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:49.907 INFO:teuthology.orchestra.run.smithi082.stderr:mgr epoch 9 is available 2024-05-08T06:28:49.907 INFO:teuthology.orchestra.run.smithi082.stderr:Generating a dashboard self-signed certificate... 2024-05-08T06:28:50.589 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:50 smithi082 ceph-mon[22957]: [08/May/2024:06:28:49] ENGINE Serving on https://172.21.15.82:7150 2024-05-08T06:28:50.589 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:50 smithi082 ceph-mon[22957]: [08/May/2024:06:28:49] ENGINE Bus STARTED 2024-05-08T06:28:50.589 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:50 smithi082 ceph-mon[22957]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-05-08T06:28:50.589 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:50 smithi082 ceph-mon[22957]: mgrmap e11: smithi082.bsnvtb(active, since 1.0575s) 2024-05-08T06:28:50.589 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:50 smithi082 ceph-mon[22957]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-05-08T06:28:50.589 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: Self-signed certificate created 2024-05-08T06:28:51.259 INFO:teuthology.orchestra.run.smithi082.stderr:Creating initial admin user... 2024-05-08T06:28:51.520 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:51 smithi082 ceph-mon[22957]: mgrmap e12: smithi082.bsnvtb(active, since 2s) 2024-05-08T06:28:51.520 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:51 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:51.521 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:51 smithi082 ceph-mon[22957]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:28:51.521 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:51 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:51.521 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:51 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:51.521 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:51 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:52.147 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$9newsQ9sLbE333b94gJFJ.EnvjKP1F6qtyejqloR/CKVmnfc/klXS", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1715149732, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-05-08T06:28:52.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:52 smithi082 ceph-mon[22957]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:28:52.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:52 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:28:52.829 INFO:teuthology.orchestra.run.smithi082.stderr:Fetching dashboard port number... 2024-05-08T06:28:53.841 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: 8443 2024-05-08T06:28:54.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:53 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3768278721' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-05-08T06:28:54.506 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: disabled 2024-05-08T06:28:54.513 INFO:teuthology.orchestra.run.smithi082.stderr:systemctl: inactive 2024-05-08T06:28:54.513 INFO:teuthology.orchestra.run.smithi082.stderr:firewalld.service is not enabled 2024-05-08T06:28:54.513 INFO:teuthology.orchestra.run.smithi082.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-05-08T06:28:54.516 INFO:teuthology.orchestra.run.smithi082.stderr:Ceph Dashboard is now available at: 2024-05-08T06:28:54.516 INFO:teuthology.orchestra.run.smithi082.stderr: 2024-05-08T06:28:54.516 INFO:teuthology.orchestra.run.smithi082.stderr: URL: https://smithi082.front.sepia.ceph.com:8443/ 2024-05-08T06:28:54.516 INFO:teuthology.orchestra.run.smithi082.stderr: User: admin 2024-05-08T06:28:54.516 INFO:teuthology.orchestra.run.smithi082.stderr: Password: jmovapuol1 2024-05-08T06:28:54.516 INFO:teuthology.orchestra.run.smithi082.stderr: 2024-05-08T06:28:54.516 INFO:teuthology.orchestra.run.smithi082.stderr:Enabling autotune for osd_memory_target 2024-05-08T06:28:56.096 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:55 smithi082 ceph-mon[22957]: mgrmap e13: smithi082.bsnvtb(active, since 6s) 2024-05-08T06:28:56.471 INFO:teuthology.orchestra.run.smithi082.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-05-08T06:28:57.111 INFO:teuthology.orchestra.run.smithi082.stderr:You can access the Ceph CLI with: 2024-05-08T06:28:57.111 INFO:teuthology.orchestra.run.smithi082.stderr: 2024-05-08T06:28:57.111 INFO:teuthology.orchestra.run.smithi082.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-05-08T06:28:57.111 INFO:teuthology.orchestra.run.smithi082.stderr: 2024-05-08T06:28:57.111 INFO:teuthology.orchestra.run.smithi082.stderr:Please consider enabling telemetry to help improve Ceph: 2024-05-08T06:28:57.111 INFO:teuthology.orchestra.run.smithi082.stderr: 2024-05-08T06:28:57.111 INFO:teuthology.orchestra.run.smithi082.stderr: ceph telemetry on 2024-05-08T06:28:57.111 INFO:teuthology.orchestra.run.smithi082.stderr: 2024-05-08T06:28:57.111 INFO:teuthology.orchestra.run.smithi082.stderr:For more information see: 2024-05-08T06:28:57.111 INFO:teuthology.orchestra.run.smithi082.stderr: 2024-05-08T06:28:57.111 INFO:teuthology.orchestra.run.smithi082.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-05-08T06:28:57.112 INFO:teuthology.orchestra.run.smithi082.stderr: 2024-05-08T06:28:57.112 INFO:teuthology.orchestra.run.smithi082.stderr:Bootstrap complete. 2024-05-08T06:28:57.152 INFO:tasks.cephadm:Fetching config... 2024-05-08T06:28:57.152 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2024-05-08T06:28:57.152 DEBUG:teuthology.orchestra.run.smithi082:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-05-08T06:28:57.169 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-05-08T06:28:57.169 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2024-05-08T06:28:57.169 DEBUG:teuthology.orchestra.run.smithi082:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-05-08T06:28:57.225 INFO:tasks.cephadm:Fetching mon keyring... 2024-05-08T06:28:57.225 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2024-05-08T06:28:57.225 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd if=/var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/keyring of=/dev/stdout 2024-05-08T06:28:57.292 INFO:tasks.cephadm:Fetching pub ssh key... 2024-05-08T06:28:57.292 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2024-05-08T06:28:57.292 DEBUG:teuthology.orchestra.run.smithi082:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-05-08T06:28:57.348 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-05-08T06:28:57.348 DEBUG:teuthology.orchestra.run.smithi082:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDYy65sFsD2B3dk2wCxff+RMU4K2nh8ABaf3mtX9vCnWduV/xQeFqqUodU+IQ24QfRLnJyscTko4er0VYrA26Iuelyj3wjE8+9qWiNX7EqCPgMHl8IZpHv2Jb/2hAF3yTQxEny0r9RMZkbGishoER4NpU1E3eR73zCs1d14Zg4pU68BBL8EaYSAZTwtLDOlKDRAPPXTjI/0qi0aOYkT1mL631hNrnBv14IpuHY8isFPj5rZeVIvuE32iALgu87dr6HzlWUtzErV2FVS7EPXH781n+LqJ0HlbtGVORuQXFnHWxT1poq8dMMDNchdWbbbIiq4J7y/T+Bszcms9ANSDUraAWp1smadukx/jbozxQONPqK0zCDk7iTgXNh5vB36Cnlg6NsspP4CS/2rb7SSR9nyy00ufcSPqPL3sxKfvU37xOCyPraxhLIvetplSqChoDSfbqa+n5kOGeu/gVdvHcVdoJsO5XsrqKpjByAQYaRlOatNj0Oas3UWxaYckH8DQOU= ceph-02181fb0-0d04-11ef-bc97-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-05-08T06:28:57.430 INFO:teuthology.orchestra.run.smithi082.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDYy65sFsD2B3dk2wCxff+RMU4K2nh8ABaf3mtX9vCnWduV/xQeFqqUodU+IQ24QfRLnJyscTko4er0VYrA26Iuelyj3wjE8+9qWiNX7EqCPgMHl8IZpHv2Jb/2hAF3yTQxEny0r9RMZkbGishoER4NpU1E3eR73zCs1d14Zg4pU68BBL8EaYSAZTwtLDOlKDRAPPXTjI/0qi0aOYkT1mL631hNrnBv14IpuHY8isFPj5rZeVIvuE32iALgu87dr6HzlWUtzErV2FVS7EPXH781n+LqJ0HlbtGVORuQXFnHWxT1poq8dMMDNchdWbbbIiq4J7y/T+Bszcms9ANSDUraAWp1smadukx/jbozxQONPqK0zCDk7iTgXNh5vB36Cnlg6NsspP4CS/2rb7SSR9nyy00ufcSPqPL3sxKfvU37xOCyPraxhLIvetplSqChoDSfbqa+n5kOGeu/gVdvHcVdoJsO5XsrqKpjByAQYaRlOatNj0Oas3UWxaYckH8DQOU= ceph-02181fb0-0d04-11ef-bc97-c7b262605968 2024-05-08T06:28:57.442 DEBUG:teuthology.orchestra.run.smithi134:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDYy65sFsD2B3dk2wCxff+RMU4K2nh8ABaf3mtX9vCnWduV/xQeFqqUodU+IQ24QfRLnJyscTko4er0VYrA26Iuelyj3wjE8+9qWiNX7EqCPgMHl8IZpHv2Jb/2hAF3yTQxEny0r9RMZkbGishoER4NpU1E3eR73zCs1d14Zg4pU68BBL8EaYSAZTwtLDOlKDRAPPXTjI/0qi0aOYkT1mL631hNrnBv14IpuHY8isFPj5rZeVIvuE32iALgu87dr6HzlWUtzErV2FVS7EPXH781n+LqJ0HlbtGVORuQXFnHWxT1poq8dMMDNchdWbbbIiq4J7y/T+Bszcms9ANSDUraAWp1smadukx/jbozxQONPqK0zCDk7iTgXNh5vB36Cnlg6NsspP4CS/2rb7SSR9nyy00ufcSPqPL3sxKfvU37xOCyPraxhLIvetplSqChoDSfbqa+n5kOGeu/gVdvHcVdoJsO5XsrqKpjByAQYaRlOatNj0Oas3UWxaYckH8DQOU= ceph-02181fb0-0d04-11ef-bc97-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-05-08T06:28:57.486 INFO:teuthology.orchestra.run.smithi134.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDYy65sFsD2B3dk2wCxff+RMU4K2nh8ABaf3mtX9vCnWduV/xQeFqqUodU+IQ24QfRLnJyscTko4er0VYrA26Iuelyj3wjE8+9qWiNX7EqCPgMHl8IZpHv2Jb/2hAF3yTQxEny0r9RMZkbGishoER4NpU1E3eR73zCs1d14Zg4pU68BBL8EaYSAZTwtLDOlKDRAPPXTjI/0qi0aOYkT1mL631hNrnBv14IpuHY8isFPj5rZeVIvuE32iALgu87dr6HzlWUtzErV2FVS7EPXH781n+LqJ0HlbtGVORuQXFnHWxT1poq8dMMDNchdWbbbIiq4J7y/T+Bszcms9ANSDUraAWp1smadukx/jbozxQONPqK0zCDk7iTgXNh5vB36Cnlg6NsspP4CS/2rb7SSR9nyy00ufcSPqPL3sxKfvU37xOCyPraxhLIvetplSqChoDSfbqa+n5kOGeu/gVdvHcVdoJsO5XsrqKpjByAQYaRlOatNj0Oas3UWxaYckH8DQOU= ceph-02181fb0-0d04-11ef-bc97-c7b262605968 2024-05-08T06:28:57.498 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-05-08T06:28:57.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:28:57 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/2274701244' entity='client.admin' 2024-05-08T06:29:00.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:00 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:00.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:00 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd/host:smithi082", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:29:00.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:00 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:00.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:00 smithi082 ceph-mon[22957]: Deploying daemon alertmanager.smithi082 on smithi082 2024-05-08T06:29:01.872 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:01 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/939963736' entity='client.admin' 2024-05-08T06:29:01.896 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-05-08T06:29:01.896 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-05-08T06:29:05.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:05 smithi082 ceph-mon[22957]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:29:05.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:05 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:05.720 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi134 2024-05-08T06:29:05.721 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-05-08T06:29:05.721 DEBUG:teuthology.orchestra.run.smithi134:> dd of=/etc/ceph/ceph.conf 2024-05-08T06:29:05.740 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-05-08T06:29:05.740 DEBUG:teuthology.orchestra.run.smithi134:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-08T06:29:05.798 INFO:tasks.cephadm:Adding host smithi134 to orchestrator... 2024-05-08T06:29:05.799 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph orch host add smithi134 2024-05-08T06:29:09.608 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:09 smithi082 ceph-mon[22957]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:29:11.623 INFO:teuthology.orchestra.run.smithi082.stdout:Added host 'smithi134' with addr '172.21.15.134' 2024-05-08T06:29:12.360 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:12 smithi082 ceph-mon[22957]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:29:12.360 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:12 smithi082 ceph-mon[22957]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi134", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:29:12.360 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:12 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:12.360 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:12 smithi082 ceph-mon[22957]: Deploying cephadm binary to smithi134 2024-05-08T06:29:12.360 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:12 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi082", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-08T06:29:12.360 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:12 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi082", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-05-08T06:29:12.360 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:12 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:29:12.361 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:12 smithi082 ceph-mon[22957]: Deploying daemon crash.smithi082 on smithi082 2024-05-08T06:29:12.361 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:12 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:13.237 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:13 smithi082 ceph-mon[22957]: Added host smithi134 2024-05-08T06:29:13.260 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph orch host ls --format=json 2024-05-08T06:29:14.361 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:14 smithi082 ceph-mon[22957]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:29:14.361 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:14 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:14.361 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:14 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:14.361 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:14 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:14.361 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:14 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:14.361 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:14 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-08T06:29:14.361 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:14 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:15.325 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:15 smithi082 ceph-mon[22957]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-08T06:29:15.325 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:15 smithi082 ceph-mon[22957]: Deploying daemon grafana.smithi082 on smithi082 2024-05-08T06:29:15.659 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:29:15.659 INFO:teuthology.orchestra.run.smithi082.stdout:[{"addr": "172.21.15.82", "hostname": "smithi082", "labels": [], "status": ""}, {"addr": "172.21.15.134", "hostname": "smithi134", "labels": [], "status": ""}] 2024-05-08T06:29:16.262 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:16 smithi082 ceph-mon[22957]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:29:16.286 INFO:tasks.cephadm:Setting crush tunables to default 2024-05-08T06:29:16.286 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph osd crush tunables default 2024-05-08T06:29:17.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:17 smithi082 ceph-mon[22957]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-08T06:29:18.256 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:18 smithi082 ceph-mon[22957]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:29:19.292 INFO:teuthology.orchestra.run.smithi082.stderr:adjusted tunables profile to default 2024-05-08T06:29:19.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:19 smithi082 ceph-mon[22957]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:29:19.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:19 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:19.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:19 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/1846518389' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-05-08T06:29:20.040 INFO:tasks.cephadm:Adding mon.smithi082 on smithi082 2024-05-08T06:29:20.040 INFO:tasks.cephadm:Adding mon.smithi134 on smithi134 2024-05-08T06:29:20.040 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph orch apply mon '2;smithi082:172.21.15.82=smithi082;smithi134:172.21.15.134=smithi134' 2024-05-08T06:29:20.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:20 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/1846518389' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-05-08T06:29:20.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:20 smithi082 ceph-mon[22957]: osdmap e4: 0 total, 0 up, 0 in 2024-05-08T06:29:21.873 INFO:teuthology.orchestra.run.smithi134.stdout:Scheduled mon update... 2024-05-08T06:29:21.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:21 smithi082 ceph-mon[22957]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:29:22.470 DEBUG:teuthology.orchestra.run.smithi134:mon.smithi134> sudo journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@mon.smithi134.service 2024-05-08T06:29:22.472 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-08T06:29:22.472 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-08T06:29:23.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:22 smithi082 ceph-mon[22957]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi082:172.21.15.82=smithi082;smithi134:172.21.15.134=smithi134", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:29:23.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:22 smithi082 ceph-mon[22957]: Saving service mon spec with placement smithi082:172.21.15.82=smithi082;smithi134:172.21.15.134=smithi134;count:2 2024-05-08T06:29:23.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:22 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:24.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:23 smithi082 ceph-mon[22957]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:29:24.302 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:29:24.302 INFO:teuthology.orchestra.run.smithi134.stdout:{"epoch":1,"fsid":"02181fb0-0d04-11ef-bc97-c7b262605968","modified":"2024-05-08T06:27:52.194153Z","created":"2024-05-08T06:27:52.194153Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi082","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:3300","nonce":0},{"type":"v1","addr":"172.21.15.82:6789","nonce":0}]},"addr":"172.21.15.82:6789/0","public_addr":"172.21.15.82:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-08T06:29:24.303 INFO:teuthology.orchestra.run.smithi134.stderr:dumped monmap epoch 1 2024-05-08T06:29:25.112 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:24 smithi082 ceph-mon[22957]: from='client.? 172.21.15.134:0/28046402' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-08T06:29:25.791 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-08T06:29:25.791 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-08T06:29:26.170 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:25 smithi082 ceph-mon[22957]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:29:27.706 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:29:27.706 INFO:teuthology.orchestra.run.smithi134.stdout:{"epoch":1,"fsid":"02181fb0-0d04-11ef-bc97-c7b262605968","modified":"2024-05-08T06:27:52.194153Z","created":"2024-05-08T06:27:52.194153Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi082","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:3300","nonce":0},{"type":"v1","addr":"172.21.15.82:6789","nonce":0}]},"addr":"172.21.15.82:6789/0","public_addr":"172.21.15.82:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-08T06:29:27.708 INFO:teuthology.orchestra.run.smithi134.stderr:dumped monmap epoch 1 2024-05-08T06:29:27.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:27 smithi082 ceph-mon[22957]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:29:27.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:27 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:27.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:27 smithi082 ceph-mon[22957]: from='client.? 172.21.15.134:0/1174141947' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-08T06:29:29.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:28 smithi082 ceph-mon[22957]: Deploying daemon node-exporter.smithi082 on smithi082 2024-05-08T06:29:29.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:28 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:29.302 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-08T06:29:29.302 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-08T06:29:30.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:29 smithi082 ceph-mon[22957]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:29:31.176 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:29:31.176 INFO:teuthology.orchestra.run.smithi134.stdout:{"epoch":1,"fsid":"02181fb0-0d04-11ef-bc97-c7b262605968","modified":"2024-05-08T06:27:52.194153Z","created":"2024-05-08T06:27:52.194153Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi082","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:3300","nonce":0},{"type":"v1","addr":"172.21.15.82:6789","nonce":0}]},"addr":"172.21.15.82:6789/0","public_addr":"172.21.15.82:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-08T06:29:31.177 INFO:teuthology.orchestra.run.smithi134.stderr:dumped monmap epoch 1 2024-05-08T06:29:31.913 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:31 smithi082 ceph-mon[22957]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:29:31.913 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:31 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:31.913 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:31 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-05-08T06:29:31.913 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:31 smithi082 ceph-mon[22957]: from='client.? 172.21.15.134:0/4124481226' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-08T06:29:32.756 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-08T06:29:32.757 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-08T06:29:32.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:32 smithi082 ceph-mon[22957]: from='mgr.14160 172.21.15.82:0/1012754186' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-05-08T06:29:32.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:32 smithi082 ceph-mon[22957]: mgrmap e14: smithi082.bsnvtb(active, since 43s) 2024-05-08T06:29:34.582 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:29:34.582 INFO:teuthology.orchestra.run.smithi134.stdout:{"epoch":1,"fsid":"02181fb0-0d04-11ef-bc97-c7b262605968","modified":"2024-05-08T06:27:52.194153Z","created":"2024-05-08T06:27:52.194153Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi082","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:3300","nonce":0},{"type":"v1","addr":"172.21.15.82:6789","nonce":0}]},"addr":"172.21.15.82:6789/0","public_addr":"172.21.15.82:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-08T06:29:34.584 INFO:teuthology.orchestra.run.smithi134.stderr:dumped monmap epoch 1 2024-05-08T06:29:34.911 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:34 smithi082 ceph-mon[22957]: from='client.? 172.21.15.134:0/3080431737' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-08T06:29:36.171 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-08T06:29:36.171 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-08T06:29:37.809 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:37 smithi082 ceph-mon[22957]: Active manager daemon smithi082.bsnvtb restarted 2024-05-08T06:29:37.809 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:37 smithi082 ceph-mon[22957]: Activating manager daemon smithi082.bsnvtb 2024-05-08T06:29:37.809 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:37 smithi082 ceph-mon[22957]: osdmap e5: 0 total, 0 up, 0 in 2024-05-08T06:29:37.992 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:29:37.992 INFO:teuthology.orchestra.run.smithi134.stdout:{"epoch":1,"fsid":"02181fb0-0d04-11ef-bc97-c7b262605968","modified":"2024-05-08T06:27:52.194153Z","created":"2024-05-08T06:27:52.194153Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi082","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:3300","nonce":0},{"type":"v1","addr":"172.21.15.82:6789","nonce":0}]},"addr":"172.21.15.82:6789/0","public_addr":"172.21.15.82:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-08T06:29:37.994 INFO:teuthology.orchestra.run.smithi134.stderr:dumped monmap epoch 1 2024-05-08T06:29:38.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:38 smithi082 ceph-mon[22957]: mgrmap e15: smithi082.bsnvtb(active, starting, since 0.0533775s) 2024-05-08T06:29:38.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi082"}]: dispatch 2024-05-08T06:29:38.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mgr metadata", "who": "smithi082.bsnvtb", "id": "smithi082.bsnvtb"}]: dispatch 2024-05-08T06:29:38.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-08T06:29:38.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-08T06:29:38.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-08T06:29:38.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:38 smithi082 ceph-mon[22957]: Manager daemon smithi082.bsnvtb is now available 2024-05-08T06:29:38.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:29:38.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:29:38.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:29:38.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:29:38.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:38.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:29:38.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:29:38.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:38 smithi082 ceph-mon[22957]: from='client.? 172.21.15.134:0/4100041105' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-08T06:29:38.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:39.549 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-08T06:29:39.550 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-08T06:29:39.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:39 smithi082 ceph-mon[22957]: [08/May/2024:06:29:38] ENGINE Bus STARTING 2024-05-08T06:29:39.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:39 smithi082 ceph-mon[22957]: [08/May/2024:06:29:38] ENGINE Serving on https://172.21.15.82:7150 2024-05-08T06:29:39.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:39 smithi082 ceph-mon[22957]: [08/May/2024:06:29:38] ENGINE Bus STARTED 2024-05-08T06:29:39.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:39 smithi082 ceph-mon[22957]: mgrmap e16: smithi082.bsnvtb(active, since 1.0615s) 2024-05-08T06:29:39.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:39 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:39.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:39 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:41.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:40 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:41.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:40 smithi082 ceph-mon[22957]: mgrmap e17: smithi082.bsnvtb(active, since 2s) 2024-05-08T06:29:41.438 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:29:41.439 INFO:teuthology.orchestra.run.smithi134.stdout:{"epoch":1,"fsid":"02181fb0-0d04-11ef-bc97-c7b262605968","modified":"2024-05-08T06:27:52.194153Z","created":"2024-05-08T06:27:52.194153Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi082","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:3300","nonce":0},{"type":"v1","addr":"172.21.15.82:6789","nonce":0}]},"addr":"172.21.15.82:6789/0","public_addr":"172.21.15.82:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-08T06:29:41.441 INFO:teuthology.orchestra.run.smithi134.stderr:dumped monmap epoch 1 2024-05-08T06:29:42.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:41 smithi082 ceph-mon[22957]: from='client.? 172.21.15.134:0/1096591292' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-08T06:29:43.738 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-08T06:29:43.738 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-08T06:29:46.080 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:46 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:46.080 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:46 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:46.080 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:46 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd/host:smithi134", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:29:46.081 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:46 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:46.180 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:29:46.181 INFO:teuthology.orchestra.run.smithi134.stdout:{"epoch":1,"fsid":"02181fb0-0d04-11ef-bc97-c7b262605968","modified":"2024-05-08T06:27:52.194153Z","created":"2024-05-08T06:27:52.194153Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi082","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:3300","nonce":0},{"type":"v1","addr":"172.21.15.82:6789","nonce":0}]},"addr":"172.21.15.82:6789/0","public_addr":"172.21.15.82:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-08T06:29:46.183 INFO:teuthology.orchestra.run.smithi134.stderr:dumped monmap epoch 1 2024-05-08T06:29:47.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:47 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:47.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:47 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd/host:smithi082", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:29:47.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:47 smithi082 ceph-mon[22957]: Updating smithi082:/etc/ceph/ceph.conf 2024-05-08T06:29:47.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:47 smithi082 ceph-mon[22957]: from='client.? 172.21.15.134:0/1554916775' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-08T06:29:47.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:47 smithi082 ceph-mon[22957]: Updating smithi082:/etc/ceph/ceph.client.admin.keyring 2024-05-08T06:29:47.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:47 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:47.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:47 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:47.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:47 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi134", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-08T06:29:47.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:47 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi134", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-05-08T06:29:47.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:47 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:29:47.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:47 smithi082 ceph-mon[22957]: Deploying daemon crash.smithi134 on smithi134 2024-05-08T06:29:47.696 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-08T06:29:47.697 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-08T06:29:49.813 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:29:49.814 INFO:teuthology.orchestra.run.smithi134.stdout:{"epoch":1,"fsid":"02181fb0-0d04-11ef-bc97-c7b262605968","modified":"2024-05-08T06:27:52.194153Z","created":"2024-05-08T06:27:52.194153Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi082","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:3300","nonce":0},{"type":"v1","addr":"172.21.15.82:6789","nonce":0}]},"addr":"172.21.15.82:6789/0","public_addr":"172.21.15.82:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-08T06:29:49.815 INFO:teuthology.orchestra.run.smithi134.stderr:dumped monmap epoch 1 2024-05-08T06:29:50.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:49 smithi082 ceph-mon[22957]: from='client.? 172.21.15.134:0/3650713100' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-08T06:29:51.460 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-08T06:29:51.460 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-08T06:29:52.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:51 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:52.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:51 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi134.zbsjta", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-08T06:29:52.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:51 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi134.zbsjta", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-08T06:29:52.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:51 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-08T06:29:52.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:51 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:29:52.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:51 smithi082 ceph-mon[22957]: Deploying daemon mgr.smithi134.zbsjta on smithi134 2024-05-08T06:29:53.904 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:29:53.905 INFO:teuthology.orchestra.run.smithi134.stdout:{"epoch":1,"fsid":"02181fb0-0d04-11ef-bc97-c7b262605968","modified":"2024-05-08T06:27:52.194153Z","created":"2024-05-08T06:27:52.194153Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi082","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:3300","nonce":0},{"type":"v1","addr":"172.21.15.82:6789","nonce":0}]},"addr":"172.21.15.82:6789/0","public_addr":"172.21.15.82:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-08T06:29:53.906 INFO:teuthology.orchestra.run.smithi134.stderr:dumped monmap epoch 1 2024-05-08T06:29:53.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:53 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:54.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:54 smithi082 ceph-mon[22957]: from='client.? 172.21.15.134:0/1794466425' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-08T06:29:54.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:54 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:54.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:54 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-08T06:29:54.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:54 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:29:55.742 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-08T06:29:55.742 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-08T06:29:55.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:55 smithi082 ceph-mon[22957]: Deploying daemon mon.smithi134 on smithi134 2024-05-08T06:29:58.910 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:29:58.910 INFO:teuthology.orchestra.run.smithi134.stdout:{"epoch":1,"fsid":"02181fb0-0d04-11ef-bc97-c7b262605968","modified":"2024-05-08T06:27:52.194153Z","created":"2024-05-08T06:27:52.194153Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi082","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:3300","nonce":0},{"type":"v1","addr":"172.21.15.82:6789","nonce":0}]},"addr":"172.21.15.82:6789/0","public_addr":"172.21.15.82:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-08T06:29:58.912 INFO:teuthology.orchestra.run.smithi134.stderr:dumped monmap epoch 1 2024-05-08T06:29:58.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:58 smithi082 ceph-mon[22957]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:29:58.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:29:58 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:29:59.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:29:58 smithi134 ceph-mon[25063]: mon.smithi134@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-05-08T06:29:59.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:29:59 smithi134 bash[25009]: 302cfd0d2bb4416dcac4ad7d5672305f76737832e63f46a3d38e9a9cf3905f72 2024-05-08T06:29:59.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:29:59 smithi134 systemd[1]: Started Ceph mon.smithi134 for 02181fb0-0d04-11ef-bc97-c7b262605968. 2024-05-08T06:30:00.738 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-08T06:30:00.738 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph mon dump -f json 2024-05-08T06:30:04.309 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi082"}]: dispatch 2024-05-08T06:30:04.309 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: mon.smithi082 calling monitor election 2024-05-08T06:30:04.309 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi134"}]: dispatch 2024-05-08T06:30:04.309 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:04.309 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi134"}]: dispatch 2024-05-08T06:30:04.309 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: from='mgr.? 172.21.15.134:0/2451743121' entity='mgr.smithi134.zbsjta' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi134.zbsjta/crt"}]: dispatch 2024-05-08T06:30:04.310 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi134"}]: dispatch 2024-05-08T06:30:04.310 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: mon.smithi134 calling monitor election 2024-05-08T06:30:04.310 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:04.310 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi134"}]: dispatch 2024-05-08T06:30:04.310 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi134"}]: dispatch 2024-05-08T06:30:04.310 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:04.310 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi134"}]: dispatch 2024-05-08T06:30:04.310 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: mon.smithi082 is new leader, mons smithi082,smithi134 in quorum (ranks 0,1) 2024-05-08T06:30:04.310 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: monmap e2: 2 mons at {smithi082=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0],smithi134=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0]} 2024-05-08T06:30:04.310 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: fsmap 2024-05-08T06:30:04.310 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: osdmap e5: 0 total, 0 up, 0 in 2024-05-08T06:30:04.310 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: mgrmap e17: smithi082.bsnvtb(active, since 26s) 2024-05-08T06:30:04.310 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: Standby manager daemon smithi134.zbsjta started 2024-05-08T06:30:04.310 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: from='mgr.? 172.21.15.134:0/2451743121' entity='mgr.smithi134.zbsjta' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-08T06:30:04.310 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: overall HEALTH_OK 2024-05-08T06:30:04.311 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: from='mgr.? 172.21.15.134:0/2451743121' entity='mgr.smithi134.zbsjta' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi134.zbsjta/key"}]: dispatch 2024-05-08T06:30:04.311 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: from='mgr.? 172.21.15.134:0/2451743121' entity='mgr.smithi134.zbsjta' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-08T06:30:04.311 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:04.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi082"}]: dispatch 2024-05-08T06:30:04.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: mon.smithi082 calling monitor election 2024-05-08T06:30:04.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi134"}]: dispatch 2024-05-08T06:30:04.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:04.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi134"}]: dispatch 2024-05-08T06:30:04.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: from='mgr.? 172.21.15.134:0/2451743121' entity='mgr.smithi134.zbsjta' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi134.zbsjta/crt"}]: dispatch 2024-05-08T06:30:04.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi134"}]: dispatch 2024-05-08T06:30:04.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: mon.smithi134 calling monitor election 2024-05-08T06:30:04.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:04.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi134"}]: dispatch 2024-05-08T06:30:04.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi134"}]: dispatch 2024-05-08T06:30:04.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:04.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi134"}]: dispatch 2024-05-08T06:30:04.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: mon.smithi082 is new leader, mons smithi082,smithi134 in quorum (ranks 0,1) 2024-05-08T06:30:04.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: monmap e2: 2 mons at {smithi082=[v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0],smithi134=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0]} 2024-05-08T06:30:04.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: fsmap 2024-05-08T06:30:04.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: osdmap e5: 0 total, 0 up, 0 in 2024-05-08T06:30:04.419 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: mgrmap e17: smithi082.bsnvtb(active, since 26s) 2024-05-08T06:30:04.419 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: Standby manager daemon smithi134.zbsjta started 2024-05-08T06:30:04.419 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: from='mgr.? 172.21.15.134:0/2451743121' entity='mgr.smithi134.zbsjta' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-08T06:30:04.419 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: overall HEALTH_OK 2024-05-08T06:30:04.419 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: from='mgr.? 172.21.15.134:0/2451743121' entity='mgr.smithi134.zbsjta' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi134.zbsjta/key"}]: dispatch 2024-05-08T06:30:04.419 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: from='mgr.? 172.21.15.134:0/2451743121' entity='mgr.smithi134.zbsjta' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-08T06:30:04.419 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:05.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:05 smithi082 ceph-mon[22957]: Deploying daemon node-exporter.smithi134 on smithi134 2024-05-08T06:30:05.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:05 smithi082 ceph-mon[22957]: mgrmap e18: smithi082.bsnvtb(active, since 26s), standbys: smithi134.zbsjta 2024-05-08T06:30:05.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:05 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mgr metadata", "who": "smithi134.zbsjta", "id": "smithi134.zbsjta"}]: dispatch 2024-05-08T06:30:05.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:05 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi134"}]: dispatch 2024-05-08T06:30:05.472 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:05 smithi134 ceph-mon[25063]: Deploying daemon node-exporter.smithi134 on smithi134 2024-05-08T06:30:05.472 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:05 smithi134 ceph-mon[25063]: mgrmap e18: smithi082.bsnvtb(active, since 26s), standbys: smithi134.zbsjta 2024-05-08T06:30:05.472 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:05 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mgr metadata", "who": "smithi134.zbsjta", "id": "smithi134.zbsjta"}]: dispatch 2024-05-08T06:30:05.472 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:05 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi134"}]: dispatch 2024-05-08T06:30:05.754 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:30:05.754 INFO:teuthology.orchestra.run.smithi134.stdout:{"epoch":2,"fsid":"02181fb0-0d04-11ef-bc97-c7b262605968","modified":"2024-05-08T06:29:59.000300Z","created":"2024-05-08T06:27:52.194153Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi082","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:3300","nonce":0},{"type":"v1","addr":"172.21.15.82:6789","nonce":0}]},"addr":"172.21.15.82:6789/0","public_addr":"172.21.15.82:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi134","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:3300","nonce":0},{"type":"v1","addr":"172.21.15.134:6789","nonce":0}]},"addr":"172.21.15.134:6789/0","public_addr":"172.21.15.134:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-05-08T06:30:05.755 INFO:teuthology.orchestra.run.smithi134.stderr:dumped monmap epoch 2 2024-05-08T06:30:06.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:06 smithi134 ceph-mon[25063]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:06.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:06 smithi134 ceph-mon[25063]: from='client.? 172.21.15.134:0/81247357' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-08T06:30:06.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:06 smithi082 ceph-mon[22957]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:06.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:06 smithi082 ceph-mon[22957]: from='client.? 172.21.15.134:0/81247357' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-08T06:30:07.094 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-05-08T06:30:07.094 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph config generate-minimal-conf 2024-05-08T06:30:08.840 INFO:teuthology.orchestra.run.smithi082.stdout:# minimal ceph.conf for 02181fb0-0d04-11ef-bc97-c7b262605968 2024-05-08T06:30:08.840 INFO:teuthology.orchestra.run.smithi082.stdout:[global] 2024-05-08T06:30:08.840 INFO:teuthology.orchestra.run.smithi082.stdout: fsid = 02181fb0-0d04-11ef-bc97-c7b262605968 2024-05-08T06:30:08.840 INFO:teuthology.orchestra.run.smithi082.stdout: mon_host = [v2:172.21.15.82:3300/0,v1:172.21.15.82:6789/0] [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] 2024-05-08T06:30:08.840 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:08 smithi082 ceph-mon[22957]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:08.840 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:08 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:08.840 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:08 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:09.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:08 smithi134 ceph-mon[25063]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:09.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:08 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:09.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:08 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:09.464 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-05-08T06:30:09.465 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2024-05-08T06:30:09.465 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd of=/etc/ceph/ceph.conf 2024-05-08T06:30:09.491 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2024-05-08T06:30:09.491 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-08T06:30:09.556 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-05-08T06:30:09.556 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/etc/ceph/ceph.conf 2024-05-08T06:30:09.586 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-05-08T06:30:09.587 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-08T06:30:09.654 INFO:tasks.cephadm:Deploying OSDs... 2024-05-08T06:30:09.654 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2024-05-08T06:30:09.654 DEBUG:teuthology.orchestra.run.smithi082:> dd if=/scratch_devs of=/dev/stdout 2024-05-08T06:30:09.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:09 smithi082 ceph-mon[22957]: Deploying daemon prometheus.smithi082 on smithi082 2024-05-08T06:30:09.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:09 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/496412053' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:30:09.670 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-05-08T06:30:09.671 DEBUG:teuthology.orchestra.run.smithi082:> stat /dev/vg_nvme/lv_1 2024-05-08T06:30:09.726 INFO:teuthology.orchestra.run.smithi082.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-05-08T06:30:09.727 INFO:teuthology.orchestra.run.smithi082.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-08T06:30:09.727 INFO:teuthology.orchestra.run.smithi082.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-05-08T06:30:09.727 INFO:teuthology.orchestra.run.smithi082.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-08T06:30:09.727 INFO:teuthology.orchestra.run.smithi082.stdout:Context: system_u:object_r:device_t:s0 2024-05-08T06:30:09.727 INFO:teuthology.orchestra.run.smithi082.stdout:Access: 2024-05-08 06:29:28.103599057 +0000 2024-05-08T06:30:09.727 INFO:teuthology.orchestra.run.smithi082.stdout:Modify: 2024-05-08 06:26:54.690028486 +0000 2024-05-08T06:30:09.727 INFO:teuthology.orchestra.run.smithi082.stdout:Change: 2024-05-08 06:26:54.690028486 +0000 2024-05-08T06:30:09.727 INFO:teuthology.orchestra.run.smithi082.stdout: Birth: 2024-05-08 06:26:54.690028486 +0000 2024-05-08T06:30:09.727 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-05-08T06:30:09.791 INFO:teuthology.orchestra.run.smithi082.stderr:1+0 records in 2024-05-08T06:30:09.791 INFO:teuthology.orchestra.run.smithi082.stderr:1+0 records out 2024-05-08T06:30:09.791 INFO:teuthology.orchestra.run.smithi082.stderr:512 bytes copied, 0.000260047 s, 2.0 MB/s 2024-05-08T06:30:09.793 DEBUG:teuthology.orchestra.run.smithi082:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-05-08T06:30:09.850 DEBUG:teuthology.orchestra.run.smithi082:> stat /dev/vg_nvme/lv_2 2024-05-08T06:30:09.906 INFO:teuthology.orchestra.run.smithi082.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-05-08T06:30:09.906 INFO:teuthology.orchestra.run.smithi082.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-08T06:30:09.906 INFO:teuthology.orchestra.run.smithi082.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-05-08T06:30:09.906 INFO:teuthology.orchestra.run.smithi082.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-08T06:30:09.906 INFO:teuthology.orchestra.run.smithi082.stdout:Context: system_u:object_r:device_t:s0 2024-05-08T06:30:09.906 INFO:teuthology.orchestra.run.smithi082.stdout:Access: 2024-05-08 06:29:28.103599057 +0000 2024-05-08T06:30:09.906 INFO:teuthology.orchestra.run.smithi082.stdout:Modify: 2024-05-08 06:26:55.041021568 +0000 2024-05-08T06:30:09.907 INFO:teuthology.orchestra.run.smithi082.stdout:Change: 2024-05-08 06:26:55.041021568 +0000 2024-05-08T06:30:09.907 INFO:teuthology.orchestra.run.smithi082.stdout: Birth: 2024-05-08 06:26:55.041021568 +0000 2024-05-08T06:30:09.907 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-05-08T06:30:09.970 INFO:teuthology.orchestra.run.smithi082.stderr:1+0 records in 2024-05-08T06:30:09.970 INFO:teuthology.orchestra.run.smithi082.stderr:1+0 records out 2024-05-08T06:30:09.970 INFO:teuthology.orchestra.run.smithi082.stderr:512 bytes copied, 0.000193207 s, 2.7 MB/s 2024-05-08T06:30:09.971 DEBUG:teuthology.orchestra.run.smithi082:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-05-08T06:30:10.028 DEBUG:teuthology.orchestra.run.smithi082:> stat /dev/vg_nvme/lv_3 2024-05-08T06:30:10.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:09 smithi134 ceph-mon[25063]: Deploying daemon prometheus.smithi082 on smithi082 2024-05-08T06:30:10.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:09 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/496412053' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:30:10.084 INFO:teuthology.orchestra.run.smithi082.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-05-08T06:30:10.084 INFO:teuthology.orchestra.run.smithi082.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-08T06:30:10.084 INFO:teuthology.orchestra.run.smithi082.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-05-08T06:30:10.084 INFO:teuthology.orchestra.run.smithi082.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-08T06:30:10.084 INFO:teuthology.orchestra.run.smithi082.stdout:Context: system_u:object_r:device_t:s0 2024-05-08T06:30:10.084 INFO:teuthology.orchestra.run.smithi082.stdout:Access: 2024-05-08 06:29:28.104599038 +0000 2024-05-08T06:30:10.084 INFO:teuthology.orchestra.run.smithi082.stdout:Modify: 2024-05-08 06:26:55.391014671 +0000 2024-05-08T06:30:10.084 INFO:teuthology.orchestra.run.smithi082.stdout:Change: 2024-05-08 06:26:55.391014671 +0000 2024-05-08T06:30:10.085 INFO:teuthology.orchestra.run.smithi082.stdout: Birth: 2024-05-08 06:26:55.391014671 +0000 2024-05-08T06:30:10.085 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-05-08T06:30:10.147 INFO:teuthology.orchestra.run.smithi082.stderr:1+0 records in 2024-05-08T06:30:10.147 INFO:teuthology.orchestra.run.smithi082.stderr:1+0 records out 2024-05-08T06:30:10.147 INFO:teuthology.orchestra.run.smithi082.stderr:512 bytes copied, 0.000177497 s, 2.9 MB/s 2024-05-08T06:30:10.148 DEBUG:teuthology.orchestra.run.smithi082:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-05-08T06:30:10.204 DEBUG:teuthology.orchestra.run.smithi082:> stat /dev/vg_nvme/lv_4 2024-05-08T06:30:10.259 INFO:teuthology.orchestra.run.smithi082.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-05-08T06:30:10.259 INFO:teuthology.orchestra.run.smithi082.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-08T06:30:10.259 INFO:teuthology.orchestra.run.smithi082.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-05-08T06:30:10.260 INFO:teuthology.orchestra.run.smithi082.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-08T06:30:10.260 INFO:teuthology.orchestra.run.smithi082.stdout:Context: system_u:object_r:device_t:s0 2024-05-08T06:30:10.260 INFO:teuthology.orchestra.run.smithi082.stdout:Access: 2024-05-08 06:29:28.104599038 +0000 2024-05-08T06:30:10.260 INFO:teuthology.orchestra.run.smithi082.stdout:Modify: 2024-05-08 06:26:55.721008168 +0000 2024-05-08T06:30:10.260 INFO:teuthology.orchestra.run.smithi082.stdout:Change: 2024-05-08 06:26:55.721008168 +0000 2024-05-08T06:30:10.260 INFO:teuthology.orchestra.run.smithi082.stdout: Birth: 2024-05-08 06:26:55.721008168 +0000 2024-05-08T06:30:10.260 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-05-08T06:30:10.323 INFO:teuthology.orchestra.run.smithi082.stderr:1+0 records in 2024-05-08T06:30:10.323 INFO:teuthology.orchestra.run.smithi082.stderr:1+0 records out 2024-05-08T06:30:10.323 INFO:teuthology.orchestra.run.smithi082.stderr:512 bytes copied, 0.000199433 s, 2.6 MB/s 2024-05-08T06:30:10.324 DEBUG:teuthology.orchestra.run.smithi082:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-05-08T06:30:10.381 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-05-08T06:30:10.381 DEBUG:teuthology.orchestra.run.smithi134:> dd if=/scratch_devs of=/dev/stdout 2024-05-08T06:30:10.400 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-05-08T06:30:10.401 DEBUG:teuthology.orchestra.run.smithi134:> stat /dev/vg_nvme/lv_1 2024-05-08T06:30:10.460 INFO:teuthology.orchestra.run.smithi134.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-05-08T06:30:10.460 INFO:teuthology.orchestra.run.smithi134.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-08T06:30:10.460 INFO:teuthology.orchestra.run.smithi134.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-05-08T06:30:10.460 INFO:teuthology.orchestra.run.smithi134.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-08T06:30:10.460 INFO:teuthology.orchestra.run.smithi134.stdout:Context: system_u:object_r:device_t:s0 2024-05-08T06:30:10.460 INFO:teuthology.orchestra.run.smithi134.stdout:Access: 2024-05-08 06:30:04.596684360 +0000 2024-05-08T06:30:10.460 INFO:teuthology.orchestra.run.smithi134.stdout:Modify: 2024-05-08 06:26:09.815472721 +0000 2024-05-08T06:30:10.460 INFO:teuthology.orchestra.run.smithi134.stdout:Change: 2024-05-08 06:26:09.815472721 +0000 2024-05-08T06:30:10.461 INFO:teuthology.orchestra.run.smithi134.stdout: Birth: 2024-05-08 06:26:09.815472721 +0000 2024-05-08T06:30:10.461 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-05-08T06:30:10.527 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records in 2024-05-08T06:30:10.527 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records out 2024-05-08T06:30:10.527 INFO:teuthology.orchestra.run.smithi134.stderr:512 bytes copied, 0.000287035 s, 1.8 MB/s 2024-05-08T06:30:10.529 DEBUG:teuthology.orchestra.run.smithi134:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-05-08T06:30:10.588 DEBUG:teuthology.orchestra.run.smithi134:> stat /dev/vg_nvme/lv_2 2024-05-08T06:30:10.647 INFO:teuthology.orchestra.run.smithi134.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-05-08T06:30:10.647 INFO:teuthology.orchestra.run.smithi134.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-08T06:30:10.647 INFO:teuthology.orchestra.run.smithi134.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-05-08T06:30:10.647 INFO:teuthology.orchestra.run.smithi134.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-08T06:30:10.647 INFO:teuthology.orchestra.run.smithi134.stdout:Context: system_u:object_r:device_t:s0 2024-05-08T06:30:10.647 INFO:teuthology.orchestra.run.smithi134.stdout:Access: 2024-05-08 06:30:04.596684360 +0000 2024-05-08T06:30:10.647 INFO:teuthology.orchestra.run.smithi134.stdout:Modify: 2024-05-08 06:26:10.174464762 +0000 2024-05-08T06:30:10.647 INFO:teuthology.orchestra.run.smithi134.stdout:Change: 2024-05-08 06:26:10.174464762 +0000 2024-05-08T06:30:10.647 INFO:teuthology.orchestra.run.smithi134.stdout: Birth: 2024-05-08 06:26:10.174464762 +0000 2024-05-08T06:30:10.648 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-05-08T06:30:10.714 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records in 2024-05-08T06:30:10.714 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records out 2024-05-08T06:30:10.714 INFO:teuthology.orchestra.run.smithi134.stderr:512 bytes copied, 0.000200019 s, 2.6 MB/s 2024-05-08T06:30:10.715 DEBUG:teuthology.orchestra.run.smithi134:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-05-08T06:30:10.774 DEBUG:teuthology.orchestra.run.smithi134:> stat /dev/vg_nvme/lv_3 2024-05-08T06:30:10.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:10 smithi134 ceph-mon[25063]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:10.794 INFO:teuthology.orchestra.run.smithi134.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-05-08T06:30:10.794 INFO:teuthology.orchestra.run.smithi134.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-08T06:30:10.794 INFO:teuthology.orchestra.run.smithi134.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-05-08T06:30:10.794 INFO:teuthology.orchestra.run.smithi134.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-08T06:30:10.794 INFO:teuthology.orchestra.run.smithi134.stdout:Context: system_u:object_r:device_t:s0 2024-05-08T06:30:10.795 INFO:teuthology.orchestra.run.smithi134.stdout:Access: 2024-05-08 06:30:04.597684338 +0000 2024-05-08T06:30:10.795 INFO:teuthology.orchestra.run.smithi134.stdout:Modify: 2024-05-08 06:26:10.496457624 +0000 2024-05-08T06:30:10.795 INFO:teuthology.orchestra.run.smithi134.stdout:Change: 2024-05-08 06:26:10.496457624 +0000 2024-05-08T06:30:10.795 INFO:teuthology.orchestra.run.smithi134.stdout: Birth: 2024-05-08 06:26:10.496457624 +0000 2024-05-08T06:30:10.795 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-05-08T06:30:10.859 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records in 2024-05-08T06:30:10.859 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records out 2024-05-08T06:30:10.859 INFO:teuthology.orchestra.run.smithi134.stderr:512 bytes copied, 0.000211168 s, 2.4 MB/s 2024-05-08T06:30:10.860 DEBUG:teuthology.orchestra.run.smithi134:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-05-08T06:30:10.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:10 smithi082 ceph-mon[22957]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:10.918 DEBUG:teuthology.orchestra.run.smithi134:> stat /dev/vg_nvme/lv_4 2024-05-08T06:30:10.974 INFO:teuthology.orchestra.run.smithi134.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-05-08T06:30:10.975 INFO:teuthology.orchestra.run.smithi134.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-08T06:30:10.975 INFO:teuthology.orchestra.run.smithi134.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-05-08T06:30:10.975 INFO:teuthology.orchestra.run.smithi134.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-08T06:30:10.975 INFO:teuthology.orchestra.run.smithi134.stdout:Context: system_u:object_r:device_t:s0 2024-05-08T06:30:10.975 INFO:teuthology.orchestra.run.smithi134.stdout:Access: 2024-05-08 06:30:04.597684338 +0000 2024-05-08T06:30:10.975 INFO:teuthology.orchestra.run.smithi134.stdout:Modify: 2024-05-08 06:26:10.854449688 +0000 2024-05-08T06:30:10.975 INFO:teuthology.orchestra.run.smithi134.stdout:Change: 2024-05-08 06:26:10.854449688 +0000 2024-05-08T06:30:10.975 INFO:teuthology.orchestra.run.smithi134.stdout: Birth: 2024-05-08 06:26:10.854449688 +0000 2024-05-08T06:30:10.976 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-05-08T06:30:11.039 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records in 2024-05-08T06:30:11.039 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records out 2024-05-08T06:30:11.039 INFO:teuthology.orchestra.run.smithi134.stderr:512 bytes copied, 0.000244959 s, 2.1 MB/s 2024-05-08T06:30:11.041 DEBUG:teuthology.orchestra.run.smithi134:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-05-08T06:30:11.098 INFO:tasks.cephadm:Deploying osd.0 on smithi082 with /dev/vg_nvme/lv_4... 2024-05-08T06:30:11.098 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-05-08T06:30:12.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:12 smithi082 ceph-mon[22957]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:12.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:12 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:13.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:12 smithi134 ceph-mon[25063]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:13.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:12 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:14.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:14 smithi082 ceph-mon[22957]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:15.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:14 smithi134 ceph-mon[25063]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:16.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:16 smithi082 ceph-mon[22957]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:17.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:16 smithi134 ceph-mon[25063]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:18.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:18 smithi082 ceph-mon[22957]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:19.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:18 smithi134 ceph-mon[25063]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:20.913 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:20 smithi082 ceph-mon[22957]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:21.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:20 smithi134 ceph-mon[25063]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:22.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:22 smithi134 ceph-mon[25063]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:22.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:22 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:22.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:22 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:30:22.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:22 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:30:22.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:22 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:30:22.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:22 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:23.083 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:22 smithi082 ceph-mon[22957]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:23.083 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:22 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:23.083 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:22 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:30:23.084 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:22 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:30:23.084 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:22 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:30:23.084 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:22 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:23.347 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:30:23.365 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi082:vg_nvme/lv_4 2024-05-08T06:30:24.939 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:24 smithi082 ceph-mon[22957]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:24.940 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:24 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:24.940 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:24 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:25.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:24 smithi134 ceph-mon[25063]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:25.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:24 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:25.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:24 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:25.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:25 smithi082 ceph-mon[22957]: Updating smithi134:/etc/ceph/ceph.conf 2024-05-08T06:30:25.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:25 smithi082 ceph-mon[22957]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-05-08T06:30:25.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:25 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-08T06:30:25.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:25 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-08T06:30:25.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:25 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:30:26.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:25 smithi134 ceph-mon[25063]: Updating smithi134:/etc/ceph/ceph.conf 2024-05-08T06:30:26.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:25 smithi134 ceph-mon[25063]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-05-08T06:30:26.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:25 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-08T06:30:26.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:25 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-08T06:30:26.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:25 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:30:27.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:26 smithi134 ceph-mon[25063]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:30:27.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:26 smithi134 ceph-mon[25063]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:27.082 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:26 smithi082 ceph-mon[22957]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:30:27.082 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:26 smithi082 ceph-mon[22957]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:27.940 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:27 smithi082 ceph-mon[22957]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:27.944 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:27 smithi134 ceph-mon[25063]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:30.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:30.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:30 smithi082 ceph-mon[22957]: Updating smithi082:/etc/ceph/ceph.conf 2024-05-08T06:30:30.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:30.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:30.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:30 smithi082 ceph-mon[22957]: Reconfiguring mon.smithi082 (unknown last config time)... 2024-05-08T06:30:30.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-08T06:30:30.419 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-08T06:30:30.419 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:30:30.419 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:30 smithi082 ceph-mon[22957]: Reconfiguring daemon mon.smithi082 on smithi082 2024-05-08T06:30:30.419 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:30 smithi082 ceph-mon[22957]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:30.419 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:30 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/1978748750' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b40392a7-c031-463b-9b63-5841f6f0ef6c"}]: dispatch 2024-05-08T06:30:30.419 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:30 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/1978748750' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b40392a7-c031-463b-9b63-5841f6f0ef6c"}]': finished 2024-05-08T06:30:30.419 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:30 smithi082 ceph-mon[22957]: osdmap e6: 1 total, 0 up, 1 in 2024-05-08T06:30:30.419 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-08T06:30:30.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:30.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:30 smithi134 ceph-mon[25063]: Updating smithi082:/etc/ceph/ceph.conf 2024-05-08T06:30:30.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:30.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:30.529 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:30 smithi134 ceph-mon[25063]: Reconfiguring mon.smithi082 (unknown last config time)... 2024-05-08T06:30:30.529 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-08T06:30:30.529 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-08T06:30:30.529 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:30:30.529 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:30 smithi134 ceph-mon[25063]: Reconfiguring daemon mon.smithi082 on smithi082 2024-05-08T06:30:30.529 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:30 smithi134 ceph-mon[25063]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:30.529 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:30 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/1978748750' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b40392a7-c031-463b-9b63-5841f6f0ef6c"}]: dispatch 2024-05-08T06:30:30.529 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:30 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/1978748750' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b40392a7-c031-463b-9b63-5841f6f0ef6c"}]': finished 2024-05-08T06:30:30.529 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:30 smithi134 ceph-mon[25063]: osdmap e6: 1 total, 0 up, 1 in 2024-05-08T06:30:30.529 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-08T06:30:31.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:31 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3920844913' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-08T06:30:31.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:31 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/3920844913' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-08T06:30:32.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:32 smithi082 ceph-mon[22957]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:32.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:32 smithi134 ceph-mon[25063]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:34.831 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:34 smithi082 ceph-mon[22957]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:34.831 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:34 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:34.831 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:34 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi082.bsnvtb", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-08T06:30:34.832 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:34 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-08T06:30:34.832 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:34 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:30:35.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:34 smithi134 ceph-mon[25063]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:35.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:34 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:35.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:34 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi082.bsnvtb", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-08T06:30:35.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:34 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-08T06:30:35.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:34 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:30:35.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:35 smithi082 ceph-mon[22957]: Reconfiguring mgr.smithi082.bsnvtb (unknown last config time)... 2024-05-08T06:30:35.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:35 smithi082 ceph-mon[22957]: Reconfiguring daemon mgr.smithi082.bsnvtb on smithi082 2024-05-08T06:30:36.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:35 smithi134 ceph-mon[25063]: Reconfiguring mgr.smithi082.bsnvtb (unknown last config time)... 2024-05-08T06:30:36.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:35 smithi134 ceph-mon[25063]: Reconfiguring daemon mgr.smithi082.bsnvtb on smithi082 2024-05-08T06:30:36.915 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:36 smithi082 ceph-mon[22957]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:36.915 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:36 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:37.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:36 smithi134 ceph-mon[25063]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:37.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:36 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:37.988 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:37 smithi082 ceph-mon[22957]: Reconfiguring alertmanager.smithi082 (dependencies changed)... 2024-05-08T06:30:37.989 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:37 smithi082 ceph-mon[22957]: Reconfiguring daemon alertmanager.smithi082 on smithi082 2024-05-08T06:30:37.989 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:30:37.989 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:30:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:37 smithi134 ceph-mon[25063]: Reconfiguring alertmanager.smithi082 (dependencies changed)... 2024-05-08T06:30:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:37 smithi134 ceph-mon[25063]: Reconfiguring daemon alertmanager.smithi082 on smithi082 2024-05-08T06:30:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:30:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:30:38.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:38 smithi082 ceph-mon[22957]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:38.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-08T06:30:38.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:30:39.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:38 smithi134 ceph-mon[25063]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:39.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:38 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-08T06:30:39.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:38 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:30:39.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:39 smithi134 ceph-mon[25063]: Deploying daemon osd.0 on smithi082 2024-05-08T06:30:39.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:39 smithi134 ceph-mon[25063]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:39.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:39 smithi082 ceph-mon[22957]: Deploying daemon osd.0 on smithi082 2024-05-08T06:30:39.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:39 smithi082 ceph-mon[22957]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:42.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:42 smithi082 ceph-mon[22957]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:43.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:42 smithi134 ceph-mon[25063]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:44.906 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:44 smithi082 ceph-mon[22957]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:44.906 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:44 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:44.906 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:44 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi082", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-08T06:30:44.906 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:44 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:30:45.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:44 smithi134 ceph-mon[25063]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:45.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:44 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:45.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:44 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi082", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-08T06:30:45.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:44 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:30:45.862 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:45 smithi082 ceph-mon[22957]: Reconfiguring crash.smithi082 (monmap changed)... 2024-05-08T06:30:45.862 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:45 smithi082 ceph-mon[22957]: Reconfiguring daemon crash.smithi082 on smithi082 2024-05-08T06:30:46.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:45 smithi134 ceph-mon[25063]: Reconfiguring crash.smithi082 (monmap changed)... 2024-05-08T06:30:46.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:45 smithi134 ceph-mon[25063]: Reconfiguring daemon crash.smithi082 on smithi082 2024-05-08T06:30:46.895 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:46 smithi082 ceph-mon[22957]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:47.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:46 smithi134 ceph-mon[25063]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:48.819 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:48 smithi082 ceph-mon[22957]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:49.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:48 smithi134 ceph-mon[25063]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:49.905 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:49 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:49.905 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:49 smithi082 ceph-mon[22957]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:50.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:49 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:50.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:49 smithi134 ceph-mon[25063]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:52.042 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:51 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:52.043 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:51 smithi082 ceph-mon[22957]: Reconfiguring grafana.smithi082 (dependencies changed)... 2024-05-08T06:30:52.043 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:51 smithi082 ceph-mon[22957]: Reconfiguring daemon grafana.smithi082 on smithi082 2024-05-08T06:30:52.043 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:51 smithi082 ceph-mon[22957]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:52.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:51 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:52.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:51 smithi134 ceph-mon[25063]: Reconfiguring grafana.smithi082 (dependencies changed)... 2024-05-08T06:30:52.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:51 smithi134 ceph-mon[25063]: Reconfiguring daemon grafana.smithi082 on smithi082 2024-05-08T06:30:52.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:51 smithi134 ceph-mon[25063]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:52.353 INFO:teuthology.orchestra.run.smithi082.stdout:Created osd(s) 0 on host 'smithi082' 2024-05-08T06:30:53.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:52 smithi082 ceph-mon[22957]: from='osd.0 [v2:172.21.15.82:6802/3933757761,v1:172.21.15.82:6803/3933757761]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-08T06:30:53.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:52 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:53.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:52 smithi134 ceph-mon[25063]: from='osd.0 [v2:172.21.15.82:6802/3933757761,v1:172.21.15.82:6803/3933757761]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-08T06:30:53.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:52 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:53.789 DEBUG:teuthology.orchestra.run.smithi082:osd.0> sudo journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.0.service 2024-05-08T06:30:53.792 INFO:tasks.cephadm:Deploying osd.1 on smithi082 with /dev/vg_nvme/lv_3... 2024-05-08T06:30:53.792 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-05-08T06:30:54.114 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:53 smithi082 ceph-mon[22957]: from='osd.0 [v2:172.21.15.82:6802/3933757761,v1:172.21.15.82:6803/3933757761]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-08T06:30:54.114 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:53 smithi082 ceph-mon[22957]: osdmap e7: 1 total, 0 up, 1 in 2024-05-08T06:30:54.114 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:53 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-08T06:30:54.114 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:53 smithi082 ceph-mon[22957]: from='osd.0 [v2:172.21.15.82:6802/3933757761,v1:172.21.15.82:6803/3933757761]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2024-05-08T06:30:54.115 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:53 smithi082 ceph-mon[22957]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:54.115 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:30:53 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-0[33372]: 2024-05-08T06:30:53.923+0000 7f0309c08700 -1 osd.0 0 waiting for initial osdmap 2024-05-08T06:30:54.115 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:30:53 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-0[33372]: 2024-05-08T06:30:53.926+0000 7f030359d700 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-08T06:30:54.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:53 smithi134 ceph-mon[25063]: from='osd.0 [v2:172.21.15.82:6802/3933757761,v1:172.21.15.82:6803/3933757761]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-08T06:30:54.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:53 smithi134 ceph-mon[25063]: osdmap e7: 1 total, 0 up, 1 in 2024-05-08T06:30:54.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:53 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-08T06:30:54.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:53 smithi134 ceph-mon[25063]: from='osd.0 [v2:172.21.15.82:6802/3933757761,v1:172.21.15.82:6803/3933757761]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2024-05-08T06:30:54.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:53 smithi134 ceph-mon[25063]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:55.052 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:54 smithi082 ceph-mon[22957]: purged_snaps scrub starts 2024-05-08T06:30:55.052 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:54 smithi082 ceph-mon[22957]: purged_snaps scrub ok 2024-05-08T06:30:55.052 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:54 smithi082 ceph-mon[22957]: from='osd.0 [v2:172.21.15.82:6802/3933757761,v1:172.21.15.82:6803/3933757761]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2024-05-08T06:30:55.052 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:54 smithi082 ceph-mon[22957]: osdmap e8: 1 total, 0 up, 1 in 2024-05-08T06:30:55.052 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:54 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-08T06:30:55.052 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:54 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-08T06:30:55.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:54 smithi134 ceph-mon[25063]: purged_snaps scrub starts 2024-05-08T06:30:55.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:54 smithi134 ceph-mon[25063]: purged_snaps scrub ok 2024-05-08T06:30:55.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:54 smithi134 ceph-mon[25063]: from='osd.0 [v2:172.21.15.82:6802/3933757761,v1:172.21.15.82:6803/3933757761]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2024-05-08T06:30:55.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:54 smithi134 ceph-mon[25063]: osdmap e8: 1 total, 0 up, 1 in 2024-05-08T06:30:55.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:54 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-08T06:30:55.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:54 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-08T06:30:56.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:55 smithi082 ceph-mon[22957]: osd.0 [v2:172.21.15.82:6802/3933757761,v1:172.21.15.82:6803/3933757761] boot 2024-05-08T06:30:56.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:55 smithi082 ceph-mon[22957]: osdmap e9: 1 total, 1 up, 1 in 2024-05-08T06:30:56.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:55 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-08T06:30:56.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:55 smithi082 ceph-mon[22957]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:56.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:55 smithi134 ceph-mon[25063]: osd.0 [v2:172.21.15.82:6802/3933757761,v1:172.21.15.82:6803/3933757761] boot 2024-05-08T06:30:56.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:55 smithi134 ceph-mon[25063]: osdmap e9: 1 total, 1 up, 1 in 2024-05-08T06:30:56.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:55 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-08T06:30:56.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:55 smithi134 ceph-mon[25063]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-08T06:30:57.089 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:56 smithi082 ceph-mon[22957]: osdmap e10: 1 total, 1 up, 1 in 2024-05-08T06:30:57.198 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:56 smithi134 ceph-mon[25063]: osdmap e10: 1 total, 1 up, 1 in 2024-05-08T06:30:58.451 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:30:58.468 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:58 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:58.469 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:58 smithi082 ceph-mon[22957]: Reconfiguring crash.smithi134 (monmap changed)... 2024-05-08T06:30:58.469 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:58 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi134", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-08T06:30:58.469 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:58 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:30:58.469 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:58 smithi082 ceph-mon[22957]: Reconfiguring daemon crash.smithi134 on smithi134 2024-05-08T06:30:58.469 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:58 smithi082 ceph-mon[22957]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:30:58.470 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi082:vg_nvme/lv_3 2024-05-08T06:30:58.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:58 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:58.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:58 smithi134 ceph-mon[25063]: Reconfiguring crash.smithi134 (monmap changed)... 2024-05-08T06:30:58.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:58 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi134", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-08T06:30:58.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:58 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:30:58.529 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:58 smithi134 ceph-mon[25063]: Reconfiguring daemon crash.smithi134 on smithi134 2024-05-08T06:30:58.529 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:58 smithi134 ceph-mon[25063]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:30:59.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:59 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:59.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:59 smithi082 ceph-mon[22957]: Reconfiguring mgr.smithi134.zbsjta (monmap changed)... 2024-05-08T06:30:59.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:59 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi134.zbsjta", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-08T06:30:59.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:59 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-08T06:30:59.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:59 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:30:59.668 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:30:59 smithi082 ceph-mon[22957]: Reconfiguring daemon mgr.smithi134.zbsjta on smithi134 2024-05-08T06:30:59.895 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:59 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:30:59.896 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:59 smithi134 ceph-mon[25063]: Reconfiguring mgr.smithi134.zbsjta (monmap changed)... 2024-05-08T06:30:59.896 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:59 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi134.zbsjta", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-08T06:30:59.896 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:59 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-08T06:30:59.896 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:59 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:30:59.896 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:30:59 smithi134 ceph-mon[25063]: Reconfiguring daemon mgr.smithi134.zbsjta on smithi134 2024-05-08T06:31:00.876 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:00 smithi082 ceph-mon[22957]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:00.876 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:00 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:00.876 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:00 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-08T06:31:00.876 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:00 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-08T06:31:00.876 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:00 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:31:00.876 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:00 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-08T06:31:00.876 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:00 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-08T06:31:00.877 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:00 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:31:00.976 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:00 smithi134 ceph-mon[25063]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:00.976 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:00 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:00.977 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:00 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-08T06:31:00.977 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:00 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-08T06:31:00.977 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:00 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:31:00.977 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:00 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-08T06:31:00.977 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:00 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-08T06:31:00.977 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:00 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:31:01.861 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:01 smithi082 ceph-mon[22957]: Reconfiguring mon.smithi134 (monmap changed)... 2024-05-08T06:31:01.861 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:01 smithi082 ceph-mon[22957]: Reconfiguring daemon mon.smithi134 on smithi134 2024-05-08T06:31:01.861 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:01 smithi082 ceph-mon[22957]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:31:01.862 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:01 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:01.862 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:01 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-08T06:31:02.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:01 smithi134 ceph-mon[25063]: Reconfiguring mon.smithi134 (monmap changed)... 2024-05-08T06:31:02.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:01 smithi134 ceph-mon[25063]: Reconfiguring daemon mon.smithi134 on smithi134 2024-05-08T06:31:02.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:01 smithi134 ceph-mon[25063]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:31:02.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:01 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:02.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:01 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-08T06:31:02.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:02 smithi082 ceph-mon[22957]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-08T06:31:02.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:02 smithi082 ceph-mon[22957]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:02.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:02 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/2560370920' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e647dd10-311f-4617-8671-b32f4c79b571"}]: dispatch 2024-05-08T06:31:02.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:02 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/2560370920' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e647dd10-311f-4617-8671-b32f4c79b571"}]': finished 2024-05-08T06:31:02.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:02 smithi082 ceph-mon[22957]: osdmap e11: 2 total, 1 up, 2 in 2024-05-08T06:31:02.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:02 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-08T06:31:03.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:02 smithi134 ceph-mon[25063]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-08T06:31:03.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:02 smithi134 ceph-mon[25063]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:03.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:02 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/2560370920' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e647dd10-311f-4617-8671-b32f4c79b571"}]: dispatch 2024-05-08T06:31:03.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:02 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/2560370920' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e647dd10-311f-4617-8671-b32f4c79b571"}]': finished 2024-05-08T06:31:03.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:02 smithi134 ceph-mon[25063]: osdmap e11: 2 total, 1 up, 2 in 2024-05-08T06:31:03.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:02 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-08T06:31:03.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:03 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/529099957' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-08T06:31:04.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:03 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/529099957' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-08T06:31:04.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:04 smithi082 ceph-mon[22957]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:05.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:04 smithi134 ceph-mon[25063]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:06.901 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:06 smithi082 ceph-mon[22957]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:07.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:06 smithi134 ceph-mon[25063]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:08.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:08 smithi134 ceph-mon[25063]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:08.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:08 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-08T06:31:08.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:08 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:31:08.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:08 smithi082 ceph-mon[22957]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:08.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:08 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-08T06:31:08.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:08 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:31:09.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:09 smithi134 ceph-mon[25063]: Deploying daemon osd.1 on smithi082 2024-05-08T06:31:09.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:09 smithi082 ceph-mon[22957]: Deploying daemon osd.1 on smithi082 2024-05-08T06:31:10.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:10 smithi082 ceph-mon[22957]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:11.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:10 smithi134 ceph-mon[25063]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:12.749 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:12 smithi082 ceph-mon[22957]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:13.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:12 smithi134 ceph-mon[25063]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:14.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:14 smithi082 ceph-mon[22957]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:14.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:14 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:15.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:14 smithi134 ceph-mon[25063]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:15.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:14 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:16.283 INFO:teuthology.orchestra.run.smithi082.stdout:Created osd(s) 1 on host 'smithi082' 2024-05-08T06:31:16.874 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:16 smithi082 ceph-mon[22957]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:16.874 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:16 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:16.874 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:16 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.82:9093"}]: dispatch 2024-05-08T06:31:16.874 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:16 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:16.874 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:16 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-08T06:31:16.874 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:16 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.82:3000"}]: dispatch 2024-05-08T06:31:16.875 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:16 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:16.875 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:16 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-08T06:31:16.875 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:16 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.82:9095"}]: dispatch 2024-05-08T06:31:16.875 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:16 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:16.875 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:16 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:31:16.875 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:16 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:31:16.875 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:16 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:31:16.898 DEBUG:teuthology.orchestra.run.smithi082:osd.1> sudo journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.1.service 2024-05-08T06:31:16.901 INFO:tasks.cephadm:Deploying osd.2 on smithi082 with /dev/vg_nvme/lv_2... 2024-05-08T06:31:16.901 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-05-08T06:31:16.963 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:16 smithi134 ceph-mon[25063]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:16.963 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:16 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:16.963 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:16 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.82:9093"}]: dispatch 2024-05-08T06:31:16.963 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:16 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:16.963 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:16 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-08T06:31:16.963 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:16 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.82:3000"}]: dispatch 2024-05-08T06:31:16.963 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:16 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:16.964 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:16 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-08T06:31:16.964 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:16 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.82:9095"}]: dispatch 2024-05-08T06:31:16.964 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:16 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:16.964 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:16 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:31:16.964 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:16 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:31:16.964 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:16 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:31:17.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:17 smithi082 ceph-mon[22957]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.82:9093"}]: dispatch 2024-05-08T06:31:17.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:17 smithi082 ceph-mon[22957]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-08T06:31:17.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:17 smithi082 ceph-mon[22957]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.82:3000"}]: dispatch 2024-05-08T06:31:17.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:17 smithi082 ceph-mon[22957]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-08T06:31:17.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:17 smithi082 ceph-mon[22957]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.82:9095"}]: dispatch 2024-05-08T06:31:17.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:17 smithi082 ceph-mon[22957]: from='osd.1 [v2:172.21.15.82:6810/2786774225,v1:172.21.15.82:6811/2786774225]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-08T06:31:18.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:17 smithi134 ceph-mon[25063]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.82:9093"}]: dispatch 2024-05-08T06:31:18.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:17 smithi134 ceph-mon[25063]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-08T06:31:18.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:17 smithi134 ceph-mon[25063]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.82:3000"}]: dispatch 2024-05-08T06:31:18.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:17 smithi134 ceph-mon[25063]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-08T06:31:18.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:17 smithi134 ceph-mon[25063]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.82:9095"}]: dispatch 2024-05-08T06:31:18.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:17 smithi134 ceph-mon[25063]: from='osd.1 [v2:172.21.15.82:6810/2786774225,v1:172.21.15.82:6811/2786774225]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-08T06:31:18.735 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:18 smithi082 ceph-mon[22957]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:18.735 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:18 smithi082 ceph-mon[22957]: from='osd.1 [v2:172.21.15.82:6810/2786774225,v1:172.21.15.82:6811/2786774225]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-08T06:31:18.735 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:18 smithi082 ceph-mon[22957]: osdmap e12: 2 total, 1 up, 2 in 2024-05-08T06:31:18.735 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:18 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-08T06:31:18.735 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:18 smithi082 ceph-mon[22957]: from='osd.1 [v2:172.21.15.82:6810/2786774225,v1:172.21.15.82:6811/2786774225]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2024-05-08T06:31:18.736 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:18 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:18.736 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:31:18 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-1[36244]: 2024-05-08T06:31:18.654+0000 7fe27be9b700 -1 osd.1 0 waiting for initial osdmap 2024-05-08T06:31:18.736 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:31:18 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-1[36244]: 2024-05-08T06:31:18.660+0000 7fe27482e700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-08T06:31:19.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:18 smithi134 ceph-mon[25063]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:19.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:18 smithi134 ceph-mon[25063]: from='osd.1 [v2:172.21.15.82:6810/2786774225,v1:172.21.15.82:6811/2786774225]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-08T06:31:19.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:18 smithi134 ceph-mon[25063]: osdmap e12: 2 total, 1 up, 2 in 2024-05-08T06:31:19.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:18 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-08T06:31:19.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:18 smithi134 ceph-mon[25063]: from='osd.1 [v2:172.21.15.82:6810/2786774225,v1:172.21.15.82:6811/2786774225]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2024-05-08T06:31:19.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:18 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:19.737 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:19 smithi082 ceph-mon[22957]: from='osd.1 [v2:172.21.15.82:6810/2786774225,v1:172.21.15.82:6811/2786774225]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2024-05-08T06:31:19.738 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:19 smithi082 ceph-mon[22957]: osdmap e13: 2 total, 1 up, 2 in 2024-05-08T06:31:19.738 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:19 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-08T06:31:19.738 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:19 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-08T06:31:19.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:19 smithi134 ceph-mon[25063]: from='osd.1 [v2:172.21.15.82:6810/2786774225,v1:172.21.15.82:6811/2786774225]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2024-05-08T06:31:19.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:19 smithi134 ceph-mon[25063]: osdmap e13: 2 total, 1 up, 2 in 2024-05-08T06:31:19.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:19 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-08T06:31:19.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:19 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-08T06:31:20.308 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:31:20.332 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi082:vg_nvme/lv_2 2024-05-08T06:31:20.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:20 smithi082 ceph-mon[22957]: purged_snaps scrub starts 2024-05-08T06:31:20.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:20 smithi082 ceph-mon[22957]: purged_snaps scrub ok 2024-05-08T06:31:20.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:20 smithi082 ceph-mon[22957]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:20.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:20 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-08T06:31:20.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:20 smithi082 ceph-mon[22957]: osd.1 [v2:172.21.15.82:6810/2786774225,v1:172.21.15.82:6811/2786774225] boot 2024-05-08T06:31:20.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:20 smithi082 ceph-mon[22957]: osdmap e14: 2 total, 2 up, 2 in 2024-05-08T06:31:20.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:20 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-08T06:31:21.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:20 smithi134 ceph-mon[25063]: purged_snaps scrub starts 2024-05-08T06:31:21.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:20 smithi134 ceph-mon[25063]: purged_snaps scrub ok 2024-05-08T06:31:21.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:20 smithi134 ceph-mon[25063]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-08T06:31:21.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:20 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-08T06:31:21.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:20 smithi134 ceph-mon[25063]: osd.1 [v2:172.21.15.82:6810/2786774225,v1:172.21.15.82:6811/2786774225] boot 2024-05-08T06:31:21.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:20 smithi134 ceph-mon[25063]: osdmap e14: 2 total, 2 up, 2 in 2024-05-08T06:31:21.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:20 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-08T06:31:22.015 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:21 smithi082 ceph-mon[22957]: osdmap e15: 2 total, 2 up, 2 in 2024-05-08T06:31:22.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:21 smithi134 ceph-mon[25063]: osdmap e15: 2 total, 2 up, 2 in 2024-05-08T06:31:22.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:22 smithi082 ceph-mon[22957]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:23.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:22 smithi134 ceph-mon[25063]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:23.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-08T06:31:23.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-08T06:31:23.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:31:23.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:24.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-08T06:31:24.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-08T06:31:24.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:31:24.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:24.779 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:24 smithi082 ceph-mon[22957]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:31:24.779 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:24 smithi082 ceph-mon[22957]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:25.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:24 smithi134 ceph-mon[25063]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:31:25.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:24 smithi134 ceph-mon[25063]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:26.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:26 smithi082 ceph-mon[22957]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:26.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:26 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/2461062444' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f0eb5a47-398a-4541-b9b4-3743c790eb31"}]: dispatch 2024-05-08T06:31:26.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:26 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/2461062444' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f0eb5a47-398a-4541-b9b4-3743c790eb31"}]': finished 2024-05-08T06:31:26.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:26 smithi082 ceph-mon[22957]: osdmap e16: 3 total, 2 up, 3 in 2024-05-08T06:31:26.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:26 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-08T06:31:26.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:26 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3720544234' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-08T06:31:27.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:26 smithi134 ceph-mon[25063]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:27.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:26 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/2461062444' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f0eb5a47-398a-4541-b9b4-3743c790eb31"}]: dispatch 2024-05-08T06:31:27.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:26 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/2461062444' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f0eb5a47-398a-4541-b9b4-3743c790eb31"}]': finished 2024-05-08T06:31:27.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:26 smithi134 ceph-mon[25063]: osdmap e16: 3 total, 2 up, 3 in 2024-05-08T06:31:27.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:26 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-08T06:31:27.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:26 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/3720544234' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-08T06:31:28.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:28 smithi082 ceph-mon[22957]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:29.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:28 smithi134 ceph-mon[25063]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:30.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:30 smithi082 ceph-mon[22957]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:30.979 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:30 smithi134 ceph-mon[25063]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:32.406 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:32 smithi082 ceph-mon[22957]: Detected new or changed devices on smithi082 2024-05-08T06:31:32.406 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:32 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:32.406 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:32 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:31:32.406 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:32 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:31:32.407 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:32 smithi082 ceph-mon[22957]: Adjusting osd_memory_target on smithi082 to 6432M 2024-05-08T06:31:32.407 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:32 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:32.407 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:32 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:32.407 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:32 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:31:32.407 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:32 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:31:32.407 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:32 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:31:32.407 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:32 smithi082 ceph-mon[22957]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:32.613 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:32 smithi134 ceph-mon[25063]: Detected new or changed devices on smithi082 2024-05-08T06:31:32.614 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:32 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:32.614 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:32 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:31:32.614 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:32 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:31:32.614 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:32 smithi134 ceph-mon[25063]: Adjusting osd_memory_target on smithi082 to 6432M 2024-05-08T06:31:32.614 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:32 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:32.614 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:32 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:32.614 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:32 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:31:32.614 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:32 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:31:32.614 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:32 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:31:32.614 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:32 smithi134 ceph-mon[25063]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:33.469 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:33 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-08T06:31:33.469 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:33 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:31:33.469 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:33 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:33.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:33 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-08T06:31:33.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:33 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:31:33.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:33 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:34.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:34 smithi082 ceph-mon[22957]: Deploying daemon osd.2 on smithi082 2024-05-08T06:31:34.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:34 smithi082 ceph-mon[22957]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:34.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:34 smithi134 ceph-mon[25063]: Deploying daemon osd.2 on smithi082 2024-05-08T06:31:34.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:34 smithi134 ceph-mon[25063]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:36.788 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:36 smithi082 ceph-mon[22957]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:37.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:36 smithi134 ceph-mon[25063]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:31:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:31:38.130 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:31:38.130 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:31:38.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:38 smithi082 ceph-mon[22957]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:38.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:39.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:38 smithi134 ceph-mon[25063]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:39.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:38 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:40.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:39 smithi134 ceph-mon[25063]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:40.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:39 smithi082 ceph-mon[22957]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:42.177 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:41 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:42.177 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:41 smithi082 ceph-mon[22957]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:42.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:41 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:42.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:41 smithi134 ceph-mon[25063]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:43.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:43 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:43.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:43 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:43.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:43 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:31:43.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:43 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:31:43.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:43 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:31:43.985 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:43 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:43.985 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:43 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:43.985 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:43 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:31:43.985 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:43 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:31:43.985 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:43 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:31:44.762 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:44 smithi082 ceph-mon[22957]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:44.762 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:44 smithi082 ceph-mon[22957]: from='osd.2 [v2:172.21.15.82:6818/2782041648,v1:172.21.15.82:6819/2782041648]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-08T06:31:44.763 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:44 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:44.799 INFO:teuthology.orchestra.run.smithi082.stdout:Created osd(s) 2 on host 'smithi082' 2024-05-08T06:31:45.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:44 smithi134 ceph-mon[25063]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:45.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:44 smithi134 ceph-mon[25063]: from='osd.2 [v2:172.21.15.82:6818/2782041648,v1:172.21.15.82:6819/2782041648]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-08T06:31:45.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:44 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:45.373 DEBUG:teuthology.orchestra.run.smithi082:osd.2> sudo journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.2.service 2024-05-08T06:31:45.375 INFO:tasks.cephadm:Deploying osd.3 on smithi082 with /dev/vg_nvme/lv_1... 2024-05-08T06:31:45.376 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-05-08T06:31:45.719 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:45 smithi082 ceph-mon[22957]: from='osd.2 [v2:172.21.15.82:6818/2782041648,v1:172.21.15.82:6819/2782041648]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-08T06:31:45.720 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:45 smithi082 ceph-mon[22957]: osdmap e17: 3 total, 2 up, 3 in 2024-05-08T06:31:45.720 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:45 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-08T06:31:45.720 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:45 smithi082 ceph-mon[22957]: from='osd.2 [v2:172.21.15.82:6818/2782041648,v1:172.21.15.82:6819/2782041648]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2024-05-08T06:31:45.720 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:45 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:45.720 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:45 smithi082 ceph-mon[22957]: from='osd.2 [v2:172.21.15.82:6818/2782041648,v1:172.21.15.82:6819/2782041648]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2024-05-08T06:31:45.720 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:45 smithi082 ceph-mon[22957]: osdmap e18: 3 total, 2 up, 3 in 2024-05-08T06:31:45.720 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:45 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-08T06:31:45.720 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:45 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-08T06:31:45.720 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:31:45 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-2[40115]: 2024-05-08T06:31:45.654+0000 7fea176e5700 -1 osd.2 0 waiting for initial osdmap 2024-05-08T06:31:45.721 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:31:45 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-2[40115]: 2024-05-08T06:31:45.657+0000 7fea14080700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-08T06:31:46.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:45 smithi134 ceph-mon[25063]: from='osd.2 [v2:172.21.15.82:6818/2782041648,v1:172.21.15.82:6819/2782041648]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-08T06:31:46.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:45 smithi134 ceph-mon[25063]: osdmap e17: 3 total, 2 up, 3 in 2024-05-08T06:31:46.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:45 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-08T06:31:46.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:45 smithi134 ceph-mon[25063]: from='osd.2 [v2:172.21.15.82:6818/2782041648,v1:172.21.15.82:6819/2782041648]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2024-05-08T06:31:46.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:45 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:46.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:45 smithi134 ceph-mon[25063]: from='osd.2 [v2:172.21.15.82:6818/2782041648,v1:172.21.15.82:6819/2782041648]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2024-05-08T06:31:46.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:45 smithi134 ceph-mon[25063]: osdmap e18: 3 total, 2 up, 3 in 2024-05-08T06:31:46.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:45 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-08T06:31:46.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:45 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-08T06:31:46.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:46 smithi082 ceph-mon[22957]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:46.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:46 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-08T06:31:47.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:46 smithi134 ceph-mon[25063]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:47.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:46 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-08T06:31:47.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:47 smithi082 ceph-mon[22957]: purged_snaps scrub starts 2024-05-08T06:31:47.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:47 smithi082 ceph-mon[22957]: purged_snaps scrub ok 2024-05-08T06:31:47.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:47 smithi082 ceph-mon[22957]: osd.2 [v2:172.21.15.82:6818/2782041648,v1:172.21.15.82:6819/2782041648] boot 2024-05-08T06:31:47.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:47 smithi082 ceph-mon[22957]: osdmap e19: 3 total, 3 up, 3 in 2024-05-08T06:31:47.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:47 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-08T06:31:48.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:47 smithi134 ceph-mon[25063]: purged_snaps scrub starts 2024-05-08T06:31:48.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:47 smithi134 ceph-mon[25063]: purged_snaps scrub ok 2024-05-08T06:31:48.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:47 smithi134 ceph-mon[25063]: osd.2 [v2:172.21.15.82:6818/2782041648,v1:172.21.15.82:6819/2782041648] boot 2024-05-08T06:31:48.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:47 smithi134 ceph-mon[25063]: osdmap e19: 3 total, 3 up, 3 in 2024-05-08T06:31:48.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:47 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-08T06:31:48.559 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:31:48.577 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi082:vg_nvme/lv_1 2024-05-08T06:31:48.989 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:48 smithi082 ceph-mon[22957]: pgmap v72: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:48.989 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:48 smithi082 ceph-mon[22957]: osdmap e20: 3 total, 3 up, 3 in 2024-05-08T06:31:48.990 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:48 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-05-08T06:31:49.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:48 smithi134 ceph-mon[25063]: pgmap v72: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-05-08T06:31:49.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:48 smithi134 ceph-mon[25063]: osdmap e20: 3 total, 3 up, 3 in 2024-05-08T06:31:49.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:48 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-05-08T06:31:49.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:49 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-05-08T06:31:49.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:49 smithi134 ceph-mon[25063]: osdmap e21: 3 total, 3 up, 3 in 2024-05-08T06:31:49.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:49 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-08T06:31:49.898 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:49 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-05-08T06:31:49.898 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:49 smithi082 ceph-mon[22957]: osdmap e21: 3 total, 3 up, 3 in 2024-05-08T06:31:49.898 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:49 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-08T06:31:50.786 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:50 smithi082 ceph-mon[22957]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-08T06:31:50.786 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:50 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-08T06:31:50.786 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:50 smithi082 ceph-mon[22957]: osdmap e22: 3 total, 3 up, 3 in 2024-05-08T06:31:50.787 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:31:50 smithi082 sudo[41490]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-08T06:31:50.787 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:31:50 smithi082 sudo[41490]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-08T06:31:50.787 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:31:50 smithi082 sudo[41490]: pam_unix(sudo:session): session closed for user root 2024-05-08T06:31:51.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:50 smithi134 ceph-mon[25063]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-08T06:31:51.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:50 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-08T06:31:51.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:50 smithi134 ceph-mon[25063]: osdmap e22: 3 total, 3 up, 3 in 2024-05-08T06:31:51.037 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:31:50 smithi082 sudo[41541]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-08T06:31:51.037 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:31:50 smithi082 sudo[41541]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-08T06:31:51.037 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:31:50 smithi082 sudo[41541]: pam_unix(sudo:session): session closed for user root 2024-05-08T06:31:51.313 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:31:51 smithi082 sudo[41560]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-08T06:31:51.313 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:31:51 smithi082 sudo[41560]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-08T06:31:51.313 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:31:51 smithi082 sudo[41560]: pam_unix(sudo:session): session closed for user root 2024-05-08T06:31:51.580 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:31:51 smithi082 sudo[41596]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-08T06:31:51.580 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:31:51 smithi082 sudo[41596]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-08T06:31:51.580 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:31:51 smithi082 sudo[41596]: pam_unix(sudo:session): session closed for user root 2024-05-08T06:31:51.851 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:31:51 smithi082 sudo[41636]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-08T06:31:51.851 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:31:51 smithi082 sudo[41636]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-08T06:31:51.851 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:31:51 smithi082 sudo[41636]: pam_unix(sudo:session): session closed for user root 2024-05-08T06:31:51.852 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:51 smithi082 ceph-mon[22957]: osdmap e23: 3 total, 3 up, 3 in 2024-05-08T06:31:51.852 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:51 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:51.852 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:51 smithi082 ceph-mon[22957]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:31:51.852 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:51 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-08T06:31:51.852 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:51 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-08T06:31:51.852 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:51 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:31:51.852 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:51 smithi082 ceph-mon[22957]: pgmap v78: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-08T06:31:52.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:51 smithi134 ceph-mon[25063]: osdmap e23: 3 total, 3 up, 3 in 2024-05-08T06:31:52.147 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:51 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:52.148 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:51 smithi134 ceph-mon[25063]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi082:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:31:52.148 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:51 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-08T06:31:52.148 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:51 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-08T06:31:52.148 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:51 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:31:52.148 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:51 smithi134 ceph-mon[25063]: pgmap v78: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-08T06:31:52.149 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:31:51 smithi082 sudo[41731]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-08T06:31:52.149 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:31:51 smithi082 sudo[41731]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-08T06:31:52.149 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:31:51 smithi082 sudo[41731]: pam_unix(sudo:session): session closed for user root 2024-05-08T06:31:52.362 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:52 smithi082 sudo[41741]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-08T06:31:52.362 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:52 smithi082 sudo[41741]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-08T06:31:52.362 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:52 smithi082 sudo[41741]: pam_unix(sudo:session): session closed for user root 2024-05-08T06:31:52.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:52 smithi082 sudo[41753]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-08T06:31:52.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:52 smithi082 sudo[41753]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-08T06:31:52.667 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:52 smithi082 sudo[41753]: pam_unix(sudo:session): session closed for user root 2024-05-08T06:31:52.861 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:52 smithi134 sudo[27378]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-08T06:31:52.861 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:52 smithi134 sudo[27378]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-08T06:31:52.862 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:52 smithi134 sudo[27378]: pam_unix(sudo:session): session closed for user root 2024-05-08T06:31:53.045 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:52 smithi082 ceph-mon[22957]: mgrmap e19: smithi082.bsnvtb(active, since 2m), standbys: smithi134.zbsjta 2024-05-08T06:31:53.045 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:52 smithi082 ceph-mon[22957]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-08T06:31:53.045 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:52 smithi082 ceph-mon[22957]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-08T06:31:53.045 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:52 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi082"}]: dispatch 2024-05-08T06:31:53.045 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:52 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi134"}]: dispatch 2024-05-08T06:31:53.045 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:52 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:53.045 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:52 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi082"}]: dispatch 2024-05-08T06:31:53.045 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:52 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi134"}]: dispatch 2024-05-08T06:31:53.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:52 smithi134 sudo[27381]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-08T06:31:53.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:52 smithi134 sudo[27381]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-08T06:31:53.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:52 smithi134 sudo[27381]: pam_unix(sudo:session): session closed for user root 2024-05-08T06:31:53.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:52 smithi134 ceph-mon[25063]: mgrmap e19: smithi082.bsnvtb(active, since 2m), standbys: smithi134.zbsjta 2024-05-08T06:31:53.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:52 smithi134 ceph-mon[25063]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-08T06:31:53.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:52 smithi134 ceph-mon[25063]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-08T06:31:53.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:52 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi082"}]: dispatch 2024-05-08T06:31:53.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:52 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi134"}]: dispatch 2024-05-08T06:31:53.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:52 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:53.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:52 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi082"}]: dispatch 2024-05-08T06:31:53.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:52 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mon metadata", "id": "smithi134"}]: dispatch 2024-05-08T06:31:54.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:53 smithi082 ceph-mon[22957]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-08T06:31:54.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:53 smithi082 ceph-mon[22957]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-08T06:31:54.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:53 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:54.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:53 smithi082 ceph-mon[22957]: pgmap v79: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-05-08T06:31:54.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:53 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/436868056' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b0ebf498-c074-4bc5-97f1-780affa07b5d"}]: dispatch 2024-05-08T06:31:54.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:53 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/436868056' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b0ebf498-c074-4bc5-97f1-780affa07b5d"}]': finished 2024-05-08T06:31:54.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:53 smithi082 ceph-mon[22957]: osdmap e24: 4 total, 3 up, 4 in 2024-05-08T06:31:54.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:53 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-08T06:31:54.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:53 smithi134 ceph-mon[25063]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-08T06:31:54.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:53 smithi134 ceph-mon[25063]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-08T06:31:54.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:53 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:31:54.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:53 smithi134 ceph-mon[25063]: pgmap v79: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-05-08T06:31:54.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:53 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/436868056' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b0ebf498-c074-4bc5-97f1-780affa07b5d"}]: dispatch 2024-05-08T06:31:54.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:53 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/436868056' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b0ebf498-c074-4bc5-97f1-780affa07b5d"}]': finished 2024-05-08T06:31:54.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:53 smithi134 ceph-mon[25063]: osdmap e24: 4 total, 3 up, 4 in 2024-05-08T06:31:54.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:53 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-08T06:31:55.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:54 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3677258437' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-08T06:31:55.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:54 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/3677258437' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-08T06:31:56.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:56 smithi082 ceph-mon[22957]: pgmap v81: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-05-08T06:31:57.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:56 smithi134 ceph-mon[25063]: pgmap v81: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-05-08T06:31:58.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:31:58 smithi082 ceph-mon[22957]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.9 KiB/s rd, 188 KiB/s wr, 12 op/s 2024-05-08T06:31:59.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:31:58 smithi134 ceph-mon[25063]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.9 KiB/s rd, 188 KiB/s wr, 12 op/s 2024-05-08T06:32:00.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:00 smithi134 ceph-mon[25063]: Detected new or changed devices on smithi082 2024-05-08T06:32:00.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:00 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:00.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:00 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:32:00.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:00 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:32:00.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:00 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:32:00.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:00 smithi134 ceph-mon[25063]: Adjusting osd_memory_target on smithi082 to 4288M 2024-05-08T06:32:00.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:00 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:00.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:00 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:00.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:00 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:32:00.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:00 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:00.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:00 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:32:00.279 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:00 smithi134 ceph-mon[25063]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 146 KiB/s wr, 10 op/s 2024-05-08T06:32:00.410 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:00 smithi082 ceph-mon[22957]: Detected new or changed devices on smithi082 2024-05-08T06:32:00.411 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:00 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:00.411 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:00 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:32:00.411 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:00 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:32:00.411 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:00 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:32:00.411 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:00 smithi082 ceph-mon[22957]: Adjusting osd_memory_target on smithi082 to 4288M 2024-05-08T06:32:00.411 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:00 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:00.411 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:00 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:00.411 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:00 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:32:00.411 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:00 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:00.411 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:00 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:32:00.411 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:00 smithi082 ceph-mon[22957]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 146 KiB/s wr, 10 op/s 2024-05-08T06:32:01.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:01 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-08T06:32:01.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:01 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:02.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:01 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-08T06:32:02.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:01 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:02.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:02 smithi082 ceph-mon[22957]: Deploying daemon osd.3 on smithi082 2024-05-08T06:32:02.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:02 smithi082 ceph-mon[22957]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-05-08T06:32:03.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:02 smithi134 ceph-mon[25063]: Deploying daemon osd.3 on smithi082 2024-05-08T06:32:03.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:02 smithi134 ceph-mon[25063]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-05-08T06:32:04.796 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:04 smithi082 ceph-mon[22957]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-05-08T06:32:05.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:04 smithi134 ceph-mon[25063]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-05-08T06:32:06.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:06 smithi082 ceph-mon[22957]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.3 KiB/s rd, 110 KiB/s wr, 7 op/s 2024-05-08T06:32:06.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:06 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:06.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:06 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:07.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:06 smithi134 ceph-mon[25063]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.3 KiB/s rd, 110 KiB/s wr, 7 op/s 2024-05-08T06:32:07.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:06 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:07.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:06 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:08.903 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:08 smithi082 ceph-mon[22957]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2024-05-08T06:32:08.904 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:08 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:08.904 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:08 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:32:08.904 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:08 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:08.904 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:08 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:32:09.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:08 smithi134 ceph-mon[25063]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2024-05-08T06:32:09.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:08 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:09.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:08 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:32:09.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:08 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:09.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:08 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:32:11.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:10 smithi134 ceph-mon[25063]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-08T06:32:11.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:10 smithi082 ceph-mon[22957]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-08T06:32:11.761 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:11 smithi082 ceph-mon[22957]: from='osd.3 [v2:172.21.15.82:6826/461300794,v1:172.21.15.82:6827/461300794]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-08T06:32:11.795 INFO:teuthology.orchestra.run.smithi082.stdout:Created osd(s) 3 on host 'smithi082' 2024-05-08T06:32:12.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:11 smithi134 ceph-mon[25063]: from='osd.3 [v2:172.21.15.82:6826/461300794,v1:172.21.15.82:6827/461300794]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-08T06:32:12.429 DEBUG:teuthology.orchestra.run.smithi082:osd.3> sudo journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.3.service 2024-05-08T06:32:12.432 INFO:tasks.cephadm:Deploying osd.4 on smithi134 with /dev/vg_nvme/lv_4... 2024-05-08T06:32:12.432 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-05-08T06:32:12.749 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:12 smithi134 ceph-mon[25063]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-08T06:32:12.749 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:12 smithi134 ceph-mon[25063]: from='osd.3 [v2:172.21.15.82:6826/461300794,v1:172.21.15.82:6827/461300794]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-08T06:32:12.750 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:12 smithi134 ceph-mon[25063]: osdmap e25: 4 total, 3 up, 4 in 2024-05-08T06:32:12.750 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:12 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-08T06:32:12.750 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:12 smithi134 ceph-mon[25063]: from='osd.3 [v2:172.21.15.82:6826/461300794,v1:172.21.15.82:6827/461300794]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2024-05-08T06:32:12.750 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:12 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:12.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:12 smithi082 ceph-mon[22957]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-08T06:32:12.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:12 smithi082 ceph-mon[22957]: from='osd.3 [v2:172.21.15.82:6826/461300794,v1:172.21.15.82:6827/461300794]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-08T06:32:12.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:12 smithi082 ceph-mon[22957]: osdmap e25: 4 total, 3 up, 4 in 2024-05-08T06:32:12.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:12 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-08T06:32:12.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:12 smithi082 ceph-mon[22957]: from='osd.3 [v2:172.21.15.82:6826/461300794,v1:172.21.15.82:6827/461300794]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]: dispatch 2024-05-08T06:32:12.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:12 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:12.918 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:32:12 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-3[43962]: 2024-05-08T06:32:12.685+0000 7fda9d13f700 -1 osd.3 0 waiting for initial osdmap 2024-05-08T06:32:12.919 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:32:12 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-3[43962]: 2024-05-08T06:32:12.690+0000 7fda962d3700 -1 osd.3 26 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-08T06:32:13.962 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:13 smithi082 ceph-mon[22957]: from='osd.3 [v2:172.21.15.82:6826/461300794,v1:172.21.15.82:6827/461300794]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2024-05-08T06:32:13.962 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:13 smithi082 ceph-mon[22957]: osdmap e26: 4 total, 3 up, 4 in 2024-05-08T06:32:13.962 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:13 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-08T06:32:13.963 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:13 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-08T06:32:14.003 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:13 smithi134 ceph-mon[25063]: from='osd.3 [v2:172.21.15.82:6826/461300794,v1:172.21.15.82:6827/461300794]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi082", "root=default"]}]': finished 2024-05-08T06:32:14.004 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:13 smithi134 ceph-mon[25063]: osdmap e26: 4 total, 3 up, 4 in 2024-05-08T06:32:14.004 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:13 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-08T06:32:14.004 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:13 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-08T06:32:14.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:14 smithi082 ceph-mon[22957]: purged_snaps scrub starts 2024-05-08T06:32:14.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:14 smithi082 ceph-mon[22957]: purged_snaps scrub ok 2024-05-08T06:32:14.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:14 smithi082 ceph-mon[22957]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-08T06:32:14.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:14 smithi082 ceph-mon[22957]: osd.3 [v2:172.21.15.82:6826/461300794,v1:172.21.15.82:6827/461300794] boot 2024-05-08T06:32:14.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:14 smithi082 ceph-mon[22957]: osdmap e27: 4 total, 4 up, 4 in 2024-05-08T06:32:14.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:14 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-08T06:32:15.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:14 smithi134 ceph-mon[25063]: purged_snaps scrub starts 2024-05-08T06:32:15.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:14 smithi134 ceph-mon[25063]: purged_snaps scrub ok 2024-05-08T06:32:15.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:14 smithi134 ceph-mon[25063]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-05-08T06:32:15.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:14 smithi134 ceph-mon[25063]: osd.3 [v2:172.21.15.82:6826/461300794,v1:172.21.15.82:6827/461300794] boot 2024-05-08T06:32:15.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:14 smithi134 ceph-mon[25063]: osdmap e27: 4 total, 4 up, 4 in 2024-05-08T06:32:15.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:14 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-08T06:32:15.138 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:32:15.158 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi134:vg_nvme/lv_4 2024-05-08T06:32:15.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:15 smithi134 ceph-mon[25063]: osdmap e28: 4 total, 4 up, 4 in 2024-05-08T06:32:15.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:15 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:15.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:15 smithi134 ceph-mon[25063]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:16.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:15 smithi082 ceph-mon[22957]: osdmap e28: 4 total, 4 up, 4 in 2024-05-08T06:32:16.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:15 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:16.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:15 smithi082 ceph-mon[22957]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:17.411 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:17 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-08T06:32:17.411 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:17 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-08T06:32:17.411 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:17 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:17.451 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:17 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-08T06:32:17.451 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:17 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-08T06:32:17.452 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:17 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:18.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:18 smithi082 ceph-mon[22957]: from='client.24135 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:32:18.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:18 smithi082 ceph-mon[22957]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:18.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:18 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:18.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:18 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:32:18.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:18 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:32:18.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:18 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:32:18.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:18 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:32:18.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:18 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:18.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:18 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:18.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:18 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:32:18.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:18 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:18.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:18 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:32:18.431 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:18 smithi134 ceph-mon[25063]: from='client.24135 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:32:18.432 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:18 smithi134 ceph-mon[25063]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:18.432 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:18 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:18.432 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:18 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:32:18.432 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:18 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:32:18.432 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:18 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:32:18.432 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:18 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:32:18.432 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:18 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:18.432 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:18 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:18.432 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:18 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:32:18.432 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:18 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:18.432 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:18 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:32:19.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:19 smithi082 ceph-mon[22957]: Adjusting osd_memory_target on smithi082 to 3216M 2024-05-08T06:32:19.473 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:19 smithi134 ceph-mon[25063]: Adjusting osd_memory_target on smithi082 to 3216M 2024-05-08T06:32:20.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:20 smithi082 ceph-mon[22957]: from='client.? 172.21.15.134:0/2976469387' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fa3c8174-2305-4238-8db5-2e328a64b9b8"}]: dispatch 2024-05-08T06:32:20.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:20 smithi082 ceph-mon[22957]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fa3c8174-2305-4238-8db5-2e328a64b9b8"}]: dispatch 2024-05-08T06:32:20.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:20 smithi082 ceph-mon[22957]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fa3c8174-2305-4238-8db5-2e328a64b9b8"}]': finished 2024-05-08T06:32:20.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:20 smithi082 ceph-mon[22957]: osdmap e29: 5 total, 4 up, 5 in 2024-05-08T06:32:20.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:20 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-08T06:32:20.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:20 smithi082 ceph-mon[22957]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:20.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:20 smithi082 ceph-mon[22957]: from='client.? 172.21.15.134:0/4237710474' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-08T06:32:20.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:20 smithi134 ceph-mon[25063]: from='client.? 172.21.15.134:0/2976469387' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fa3c8174-2305-4238-8db5-2e328a64b9b8"}]: dispatch 2024-05-08T06:32:20.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:20 smithi134 ceph-mon[25063]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fa3c8174-2305-4238-8db5-2e328a64b9b8"}]: dispatch 2024-05-08T06:32:20.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:20 smithi134 ceph-mon[25063]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fa3c8174-2305-4238-8db5-2e328a64b9b8"}]': finished 2024-05-08T06:32:20.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:20 smithi134 ceph-mon[25063]: osdmap e29: 5 total, 4 up, 5 in 2024-05-08T06:32:20.529 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:20 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-08T06:32:20.529 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:20 smithi134 ceph-mon[25063]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:20.529 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:20 smithi134 ceph-mon[25063]: from='client.? 172.21.15.134:0/4237710474' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-08T06:32:22.848 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:22 smithi134 ceph-mon[25063]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:22.849 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:22 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:22.849 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:22 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:22.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:22 smithi082 ceph-mon[22957]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:22.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:22 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:22.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:22 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:24.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:24 smithi134 ceph-mon[25063]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:24.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:24 smithi082 ceph-mon[22957]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:25.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:25 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-08T06:32:25.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:25 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:25.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:25 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-08T06:32:25.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:25 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:26.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:26 smithi082 ceph-mon[22957]: Deploying daemon osd.4 on smithi134 2024-05-08T06:32:26.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:26 smithi082 ceph-mon[22957]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:26.939 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:26 smithi134 ceph-mon[25063]: Deploying daemon osd.4 on smithi134 2024-05-08T06:32:26.939 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:26 smithi134 ceph-mon[25063]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:28.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:28 smithi082 ceph-mon[22957]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:28.984 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:28 smithi134 ceph-mon[25063]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:30.893 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:30 smithi082 ceph-mon[22957]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:30.893 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:30.893 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:32:30.893 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:30.893 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:32:30.955 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:30 smithi134 ceph-mon[25063]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:30.955 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:30.955 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:32:30.955 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:30.955 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:32:32.969 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:32 smithi134 ceph-mon[25063]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:33.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:32 smithi082 ceph-mon[22957]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:34.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:33 smithi134 ceph-mon[25063]: from='osd.4 [v2:172.21.15.134:6800/2685273941,v1:172.21.15.134:6801/2685273941]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-08T06:32:34.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:33 smithi134 ceph-mon[25063]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-08T06:32:34.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:33 smithi082 ceph-mon[22957]: from='osd.4 [v2:172.21.15.134:6800/2685273941,v1:172.21.15.134:6801/2685273941]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-08T06:32:34.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:33 smithi082 ceph-mon[22957]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-08T06:32:34.181 INFO:teuthology.orchestra.run.smithi134.stdout:Created osd(s) 4 on host 'smithi134' 2024-05-08T06:32:34.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:34 smithi082 ceph-mon[22957]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:34.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:34 smithi082 ceph-mon[22957]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-08T06:32:34.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:34 smithi082 ceph-mon[22957]: osdmap e30: 5 total, 4 up, 5 in 2024-05-08T06:32:34.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:34 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-08T06:32:34.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:34 smithi082 ceph-mon[22957]: from='osd.4 [v2:172.21.15.134:6800/2685273941,v1:172.21.15.134:6801/2685273941]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-05-08T06:32:34.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:34 smithi082 ceph-mon[22957]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-05-08T06:32:34.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:34 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:35.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:34 smithi134 ceph-mon[25063]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:35.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:34 smithi134 ceph-mon[25063]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-08T06:32:35.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:34 smithi134 ceph-mon[25063]: osdmap e30: 5 total, 4 up, 5 in 2024-05-08T06:32:35.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:34 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-08T06:32:35.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:34 smithi134 ceph-mon[25063]: from='osd.4 [v2:172.21.15.134:6800/2685273941,v1:172.21.15.134:6801/2685273941]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-05-08T06:32:35.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:34 smithi134 ceph-mon[25063]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-05-08T06:32:35.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:34 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:35.089 DEBUG:teuthology.orchestra.run.smithi134:osd.4> sudo journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.4.service 2024-05-08T06:32:35.092 INFO:tasks.cephadm:Deploying osd.5 on smithi134 with /dev/vg_nvme/lv_3... 2024-05-08T06:32:35.092 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-05-08T06:32:36.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:35 smithi134 ceph-mon[25063]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-05-08T06:32:36.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:35 smithi134 ceph-mon[25063]: osdmap e31: 5 total, 4 up, 5 in 2024-05-08T06:32:36.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:35 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-08T06:32:36.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:35 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-08T06:32:36.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:35 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:36.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:35 smithi134 ceph-mon[25063]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:36.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:35 smithi134 ceph-mon[25063]: osd.4 [v2:172.21.15.134:6800/2685273941,v1:172.21.15.134:6801/2685273941] boot 2024-05-08T06:32:36.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:35 smithi134 ceph-mon[25063]: osdmap e32: 5 total, 5 up, 5 in 2024-05-08T06:32:36.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:35 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-08T06:32:36.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:35 smithi082 ceph-mon[22957]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-05-08T06:32:36.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:35 smithi082 ceph-mon[22957]: osdmap e31: 5 total, 4 up, 5 in 2024-05-08T06:32:36.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:35 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-08T06:32:36.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:35 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-08T06:32:36.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:35 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:36.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:35 smithi082 ceph-mon[22957]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-08T06:32:36.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:35 smithi082 ceph-mon[22957]: osd.4 [v2:172.21.15.134:6800/2685273941,v1:172.21.15.134:6801/2685273941] boot 2024-05-08T06:32:36.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:35 smithi082 ceph-mon[22957]: osdmap e32: 5 total, 5 up, 5 in 2024-05-08T06:32:36.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:35 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-08T06:32:37.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:37 smithi082 ceph-mon[22957]: purged_snaps scrub starts 2024-05-08T06:32:37.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:37 smithi082 ceph-mon[22957]: purged_snaps scrub ok 2024-05-08T06:32:37.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:37 smithi082 ceph-mon[22957]: osdmap e33: 5 total, 5 up, 5 in 2024-05-08T06:32:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:37 smithi134 ceph-mon[25063]: purged_snaps scrub starts 2024-05-08T06:32:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:37 smithi134 ceph-mon[25063]: purged_snaps scrub ok 2024-05-08T06:32:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:37 smithi134 ceph-mon[25063]: osdmap e33: 5 total, 5 up, 5 in 2024-05-08T06:32:38.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:38 smithi134 ceph-mon[25063]: pgmap v111: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-08T06:32:38.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:38 smithi134 ceph-mon[25063]: osdmap e34: 5 total, 5 up, 5 in 2024-05-08T06:32:38.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:38 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:32:38.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:38 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:32:38.787 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:32:38.806 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi134:vg_nvme/lv_3 2024-05-08T06:32:39.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:38 smithi082 ceph-mon[22957]: pgmap v111: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-08T06:32:39.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:38 smithi082 ceph-mon[22957]: osdmap e34: 5 total, 5 up, 5 in 2024-05-08T06:32:39.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:32:39.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:32:39.710 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:39 smithi134 ceph-mon[25063]: osdmap e35: 5 total, 5 up, 5 in 2024-05-08T06:32:40.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:39 smithi082 ceph-mon[22957]: osdmap e35: 5 total, 5 up, 5 in 2024-05-08T06:32:40.969 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:40 smithi134 ceph-mon[25063]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-08T06:32:40.970 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:40 smithi134 ceph-mon[25063]: osdmap e36: 5 total, 5 up, 5 in 2024-05-08T06:32:40.970 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:40 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:40.970 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:40 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:32:40.970 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:40 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:40.970 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:40 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:40.970 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:40 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:32:40.970 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:40 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:40.970 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:40 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:32:40.970 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:40 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:41.068 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:40 smithi082 ceph-mon[22957]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-08T06:32:41.069 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:40 smithi082 ceph-mon[22957]: osdmap e36: 5 total, 5 up, 5 in 2024-05-08T06:32:41.069 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:40 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:41.069 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:40 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:32:41.069 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:40 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:41.069 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:40 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:41.069 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:40 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:32:41.069 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:40 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:41.069 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:40 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:32:41.069 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:40 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:42.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:41 smithi134 ceph-mon[25063]: Detected new or changed devices on smithi134 2024-05-08T06:32:42.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:41 smithi134 ceph-mon[25063]: Adjusting osd_memory_target on smithi134 to 15936M 2024-05-08T06:32:42.080 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:41 smithi082 ceph-mon[22957]: Detected new or changed devices on smithi134 2024-05-08T06:32:42.080 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:41 smithi082 ceph-mon[22957]: Adjusting osd_memory_target on smithi134 to 15936M 2024-05-08T06:32:42.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:42 smithi082 ceph-mon[22957]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 104 KiB/s, 0 objects/s recovering 2024-05-08T06:32:42.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:42 smithi082 ceph-mon[22957]: from='client.24153 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:32:42.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:42 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-08T06:32:42.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:42 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-08T06:32:42.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:42 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:42.945 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:42 smithi134 ceph-mon[25063]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 104 KiB/s, 0 objects/s recovering 2024-05-08T06:32:42.945 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:42 smithi134 ceph-mon[25063]: from='client.24153 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:32:42.945 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:42 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-08T06:32:42.945 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:42 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-08T06:32:42.945 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:42 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:44.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:44 smithi082 ceph-mon[22957]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-08T06:32:44.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:44 smithi082 ceph-mon[22957]: from='client.? 172.21.15.134:0/1393828284' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "37d8326a-b22d-41a3-aed4-5999fd2a0d09"}]: dispatch 2024-05-08T06:32:44.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:44 smithi082 ceph-mon[22957]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "37d8326a-b22d-41a3-aed4-5999fd2a0d09"}]: dispatch 2024-05-08T06:32:44.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:44 smithi082 ceph-mon[22957]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "37d8326a-b22d-41a3-aed4-5999fd2a0d09"}]': finished 2024-05-08T06:32:44.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:44 smithi082 ceph-mon[22957]: osdmap e37: 6 total, 5 up, 6 in 2024-05-08T06:32:44.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:44 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-08T06:32:44.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:44 smithi082 ceph-mon[22957]: from='client.? 172.21.15.134:0/2018254958' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-08T06:32:45.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:44 smithi134 ceph-mon[25063]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-08T06:32:45.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:44 smithi134 ceph-mon[25063]: from='client.? 172.21.15.134:0/1393828284' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "37d8326a-b22d-41a3-aed4-5999fd2a0d09"}]: dispatch 2024-05-08T06:32:45.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:44 smithi134 ceph-mon[25063]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "37d8326a-b22d-41a3-aed4-5999fd2a0d09"}]: dispatch 2024-05-08T06:32:45.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:44 smithi134 ceph-mon[25063]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "37d8326a-b22d-41a3-aed4-5999fd2a0d09"}]': finished 2024-05-08T06:32:45.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:44 smithi134 ceph-mon[25063]: osdmap e37: 6 total, 5 up, 6 in 2024-05-08T06:32:45.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:44 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-08T06:32:45.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:44 smithi134 ceph-mon[25063]: from='client.? 172.21.15.134:0/2018254958' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-08T06:32:46.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:46 smithi082 ceph-mon[22957]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-05-08T06:32:47.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:46 smithi134 ceph-mon[25063]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-05-08T06:32:48.692 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:48 smithi134 ceph-mon[25063]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-08T06:32:48.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:48 smithi082 ceph-mon[22957]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-08T06:32:50.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:50 smithi134 ceph-mon[25063]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-08T06:32:50.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:50 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-08T06:32:50.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:50 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:51.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:50 smithi082 ceph-mon[22957]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-08T06:32:51.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:50 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-08T06:32:51.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:50 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:51.911 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:51 smithi134 ceph-mon[25063]: Deploying daemon osd.5 on smithi134 2024-05-08T06:32:52.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:51 smithi082 ceph-mon[22957]: Deploying daemon osd.5 on smithi134 2024-05-08T06:32:52.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:52 smithi134 ceph-mon[25063]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-08T06:32:53.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:52 smithi082 ceph-mon[22957]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-08T06:32:54.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:54 smithi134 ceph-mon[25063]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-08T06:32:54.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:54 smithi082 ceph-mon[22957]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-08T06:32:56.418 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:56 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:56.419 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:56 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:32:56.419 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:56 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:56.419 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:56 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:32:56.419 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:56 smithi082 ceph-mon[22957]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-08T06:32:56.480 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:56 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:56.481 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:56 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:32:56.481 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:56 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:32:56.481 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:56 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:32:56.481 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:56 smithi134 ceph-mon[25063]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-08T06:32:58.885 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:58 smithi134 ceph-mon[25063]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-08T06:32:58.885 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:58 smithi134 ceph-mon[25063]: from='osd.5 [v2:172.21.15.134:6808/1347883706,v1:172.21.15.134:6809/1347883706]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-08T06:32:58.886 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:58 smithi134 ceph-mon[25063]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-08T06:32:58.918 INFO:teuthology.orchestra.run.smithi134.stdout:Created osd(s) 5 on host 'smithi134' 2024-05-08T06:32:59.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:58 smithi082 ceph-mon[22957]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-08T06:32:59.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:58 smithi082 ceph-mon[22957]: from='osd.5 [v2:172.21.15.134:6808/1347883706,v1:172.21.15.134:6809/1347883706]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-08T06:32:59.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:58 smithi082 ceph-mon[22957]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-08T06:32:59.537 DEBUG:teuthology.orchestra.run.smithi134:osd.5> sudo journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.5.service 2024-05-08T06:32:59.539 INFO:tasks.cephadm:Deploying osd.6 on smithi134 with /dev/vg_nvme/lv_2... 2024-05-08T06:32:59.540 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-05-08T06:32:59.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:59 smithi134 ceph-mon[25063]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-08T06:32:59.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:59 smithi134 ceph-mon[25063]: osdmap e38: 6 total, 5 up, 6 in 2024-05-08T06:32:59.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:59 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-08T06:32:59.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:59 smithi134 ceph-mon[25063]: from='osd.5 [v2:172.21.15.134:6808/1347883706,v1:172.21.15.134:6809/1347883706]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-05-08T06:32:59.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:59 smithi134 ceph-mon[25063]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-05-08T06:32:59.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:59 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:59.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:59 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:32:59.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:59 smithi134 ceph-mon[25063]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-08T06:32:59.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:59 smithi134 ceph-mon[25063]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-05-08T06:32:59.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:59 smithi134 ceph-mon[25063]: osdmap e39: 6 total, 5 up, 6 in 2024-05-08T06:32:59.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:32:59 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-08T06:32:59.779 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:32:59 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:32:59.705+0000 7f803b49c700 -1 osd.5 0 waiting for initial osdmap 2024-05-08T06:32:59.779 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:32:59 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:32:59.710+0000 7f8037e37700 -1 osd.5 39 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-08T06:33:00.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:59 smithi082 ceph-mon[22957]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-08T06:33:00.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:59 smithi082 ceph-mon[22957]: osdmap e38: 6 total, 5 up, 6 in 2024-05-08T06:33:00.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:59 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-08T06:33:00.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:59 smithi082 ceph-mon[22957]: from='osd.5 [v2:172.21.15.134:6808/1347883706,v1:172.21.15.134:6809/1347883706]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-05-08T06:33:00.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:59 smithi082 ceph-mon[22957]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-05-08T06:33:00.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:59 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:00.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:59 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:00.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:59 smithi082 ceph-mon[22957]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-08T06:33:00.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:59 smithi082 ceph-mon[22957]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-05-08T06:33:00.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:59 smithi082 ceph-mon[22957]: osdmap e39: 6 total, 5 up, 6 in 2024-05-08T06:33:00.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:32:59 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-08T06:33:00.796 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:00 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-08T06:33:00.796 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:00 smithi134 ceph-mon[25063]: osd.5 [v2:172.21.15.134:6808/1347883706,v1:172.21.15.134:6809/1347883706] boot 2024-05-08T06:33:00.796 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:00 smithi134 ceph-mon[25063]: osdmap e40: 6 total, 6 up, 6 in 2024-05-08T06:33:00.797 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:00 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-08T06:33:01.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:00 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-08T06:33:01.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:00 smithi082 ceph-mon[22957]: osd.5 [v2:172.21.15.134:6808/1347883706,v1:172.21.15.134:6809/1347883706] boot 2024-05-08T06:33:01.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:00 smithi082 ceph-mon[22957]: osdmap e40: 6 total, 6 up, 6 in 2024-05-08T06:33:01.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:00 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-08T06:33:02.026 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:01 smithi134 ceph-mon[25063]: purged_snaps scrub starts 2024-05-08T06:33:02.026 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:01 smithi134 ceph-mon[25063]: purged_snaps scrub ok 2024-05-08T06:33:02.026 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:01 smithi134 ceph-mon[25063]: pgmap v130: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-08T06:33:02.026 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:01 smithi134 ceph-mon[25063]: osdmap e41: 6 total, 6 up, 6 in 2024-05-08T06:33:02.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:01 smithi082 ceph-mon[22957]: purged_snaps scrub starts 2024-05-08T06:33:02.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:01 smithi082 ceph-mon[22957]: purged_snaps scrub ok 2024-05-08T06:33:02.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:01 smithi082 ceph-mon[22957]: pgmap v130: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-08T06:33:02.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:01 smithi082 ceph-mon[22957]: osdmap e41: 6 total, 6 up, 6 in 2024-05-08T06:33:02.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:02 smithi134 ceph-mon[25063]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-05-08T06:33:02.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:02 smithi134 ceph-mon[25063]: osdmap e42: 6 total, 6 up, 6 in 2024-05-08T06:33:03.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:02 smithi082 ceph-mon[22957]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-05-08T06:33:03.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:02 smithi082 ceph-mon[22957]: osdmap e42: 6 total, 6 up, 6 in 2024-05-08T06:33:03.265 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:33:03.286 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi134:vg_nvme/lv_2 2024-05-08T06:33:03.882 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:03 smithi134 ceph-mon[25063]: pgmap v133: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-08T06:33:04.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:03 smithi082 ceph-mon[22957]: pgmap v133: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-08T06:33:04.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:04 smithi082 ceph-mon[22957]: osdmap e43: 6 total, 6 up, 6 in 2024-05-08T06:33:04.975 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:04 smithi134 ceph-mon[25063]: osdmap e43: 6 total, 6 up, 6 in 2024-05-08T06:33:05.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:05 smithi134 ceph-mon[25063]: osdmap e44: 6 total, 6 up, 6 in 2024-05-08T06:33:05.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:05 smithi134 ceph-mon[25063]: Detected new or changed devices on smithi134 2024-05-08T06:33:05.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:05 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:05.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:05 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:33:05.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:05 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:33:05.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:05 smithi134 ceph-mon[25063]: Adjusting osd_memory_target on smithi134 to 7968M 2024-05-08T06:33:05.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:05 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:05.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:05 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:05.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:05 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:33:05.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:05 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:33:05.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:05 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:33:05.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:05 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:05.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:05 smithi134 ceph-mon[25063]: pgmap v136: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-08T06:33:06.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:05 smithi082 ceph-mon[22957]: osdmap e44: 6 total, 6 up, 6 in 2024-05-08T06:33:06.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:05 smithi082 ceph-mon[22957]: Detected new or changed devices on smithi134 2024-05-08T06:33:06.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:05 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:06.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:05 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:33:06.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:05 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:33:06.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:05 smithi082 ceph-mon[22957]: Adjusting osd_memory_target on smithi134 to 7968M 2024-05-08T06:33:06.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:05 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:06.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:05 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:06.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:05 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:33:06.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:05 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:33:06.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:05 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:33:06.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:05 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:06.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:05 smithi082 ceph-mon[22957]: pgmap v136: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-08T06:33:07.408 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:07 smithi134 ceph-mon[25063]: from='client.24171 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:33:07.409 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:07 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-08T06:33:07.409 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:07 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-08T06:33:07.409 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:07 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:33:07.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:07 smithi082 ceph-mon[22957]: from='client.24171 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:33:07.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:07 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-08T06:33:07.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:07 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-08T06:33:07.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:07 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:33:08.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:08 smithi082 ceph-mon[22957]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-05-08T06:33:08.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:08 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:08.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:08 smithi082 ceph-mon[22957]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-05-08T06:33:08.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:08 smithi082 ceph-mon[22957]: Cluster is now healthy 2024-05-08T06:33:08.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:08 smithi082 ceph-mon[22957]: from='client.? 172.21.15.134:0/2354864391' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "67965443-21f8-49ce-817e-27360f3cd976"}]: dispatch 2024-05-08T06:33:08.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:08 smithi082 ceph-mon[22957]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "67965443-21f8-49ce-817e-27360f3cd976"}]: dispatch 2024-05-08T06:33:08.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:08 smithi082 ceph-mon[22957]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "67965443-21f8-49ce-817e-27360f3cd976"}]': finished 2024-05-08T06:33:08.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:08 smithi082 ceph-mon[22957]: osdmap e45: 7 total, 6 up, 7 in 2024-05-08T06:33:08.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:08 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-08T06:33:09.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:08 smithi134 ceph-mon[25063]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-05-08T06:33:09.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:08 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:09.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:08 smithi134 ceph-mon[25063]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-05-08T06:33:09.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:08 smithi134 ceph-mon[25063]: Cluster is now healthy 2024-05-08T06:33:09.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:08 smithi134 ceph-mon[25063]: from='client.? 172.21.15.134:0/2354864391' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "67965443-21f8-49ce-817e-27360f3cd976"}]: dispatch 2024-05-08T06:33:09.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:08 smithi134 ceph-mon[25063]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "67965443-21f8-49ce-817e-27360f3cd976"}]: dispatch 2024-05-08T06:33:09.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:08 smithi134 ceph-mon[25063]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "67965443-21f8-49ce-817e-27360f3cd976"}]': finished 2024-05-08T06:33:09.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:08 smithi134 ceph-mon[25063]: osdmap e45: 7 total, 6 up, 7 in 2024-05-08T06:33:09.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:08 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-08T06:33:09.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:09 smithi134 ceph-mon[25063]: from='client.? 172.21.15.134:0/1629609154' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-08T06:33:09.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:09 smithi082 ceph-mon[22957]: from='client.? 172.21.15.134:0/1629609154' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-08T06:33:10.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:10 smithi082 ceph-mon[22957]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-08T06:33:11.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:10 smithi134 ceph-mon[25063]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-08T06:33:12.952 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:12 smithi134 ceph-mon[25063]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-08T06:33:13.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:12 smithi082 ceph-mon[22957]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-08T06:33:14.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:14 smithi082 ceph-mon[22957]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-08T06:33:15.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:14 smithi134 ceph-mon[25063]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-08T06:33:15.934 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:15 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-08T06:33:15.934 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:15 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:33:15.934 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:15 smithi134 ceph-mon[25063]: Deploying daemon osd.6 on smithi134 2024-05-08T06:33:16.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:15 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-08T06:33:16.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:15 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:33:16.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:15 smithi082 ceph-mon[22957]: Deploying daemon osd.6 on smithi134 2024-05-08T06:33:17.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:16 smithi134 ceph-mon[25063]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-08T06:33:17.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:16 smithi082 ceph-mon[22957]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-08T06:33:19.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:18 smithi134 ceph-mon[25063]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-08T06:33:19.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:18 smithi082 ceph-mon[22957]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-08T06:33:21.010 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:20 smithi134 ceph-mon[25063]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-08T06:33:21.011 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:20 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:21.011 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:20 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:33:21.011 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:20 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:33:21.011 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:20 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:33:21.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:20 smithi082 ceph-mon[22957]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-08T06:33:21.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:20 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:21.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:20 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:33:21.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:20 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:33:21.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:20 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:33:22.948 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:22 smithi134 ceph-mon[25063]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-08T06:33:22.948 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:22 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:23.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:22 smithi082 ceph-mon[22957]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-08T06:33:23.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:22 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:23.230 INFO:teuthology.orchestra.run.smithi134.stdout:Created osd(s) 6 on host 'smithi134' 2024-05-08T06:33:23.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:23 smithi134 ceph-mon[25063]: from='osd.6 [v2:172.21.15.134:6816/513779403,v1:172.21.15.134:6817/513779403]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-08T06:33:23.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:23 smithi134 ceph-mon[25063]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-08T06:33:23.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:23.808 DEBUG:teuthology.orchestra.run.smithi134:osd.6> sudo journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.6.service 2024-05-08T06:33:23.810 INFO:tasks.cephadm:Deploying osd.7 on smithi134 with /dev/vg_nvme/lv_1... 2024-05-08T06:33:23.810 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-05-08T06:33:24.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:23 smithi082 ceph-mon[22957]: from='osd.6 [v2:172.21.15.134:6816/513779403,v1:172.21.15.134:6817/513779403]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-08T06:33:24.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:23 smithi082 ceph-mon[22957]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-08T06:33:24.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:24.883 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:24 smithi134 ceph-mon[25063]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-08T06:33:24.883 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:24 smithi134 ceph-mon[25063]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-08T06:33:24.883 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:24 smithi134 ceph-mon[25063]: osdmap e46: 7 total, 6 up, 7 in 2024-05-08T06:33:24.883 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:24 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-08T06:33:24.883 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:24 smithi134 ceph-mon[25063]: from='osd.6 [v2:172.21.15.134:6816/513779403,v1:172.21.15.134:6817/513779403]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-05-08T06:33:24.883 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:24 smithi134 ceph-mon[25063]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-05-08T06:33:24.884 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:33:24 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:33:24.710+0000 7f0ae9a81700 -1 osd.6 0 waiting for initial osdmap 2024-05-08T06:33:24.884 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:33:24 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:33:24.717+0000 7f0ae3416700 -1 osd.6 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-08T06:33:24.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:24 smithi082 ceph-mon[22957]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-08T06:33:24.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:24 smithi082 ceph-mon[22957]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-08T06:33:24.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:24 smithi082 ceph-mon[22957]: osdmap e46: 7 total, 6 up, 7 in 2024-05-08T06:33:24.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:24 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-08T06:33:24.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:24 smithi082 ceph-mon[22957]: from='osd.6 [v2:172.21.15.134:6816/513779403,v1:172.21.15.134:6817/513779403]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-05-08T06:33:24.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:24 smithi082 ceph-mon[22957]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-05-08T06:33:26.021 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:25 smithi134 ceph-mon[25063]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-05-08T06:33:26.021 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:25 smithi134 ceph-mon[25063]: osdmap e47: 7 total, 6 up, 7 in 2024-05-08T06:33:26.021 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:25 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-08T06:33:26.021 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:25 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-08T06:33:26.021 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:25 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:26.021 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:25 smithi134 ceph-mon[25063]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-08T06:33:26.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:25 smithi082 ceph-mon[22957]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-05-08T06:33:26.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:25 smithi082 ceph-mon[22957]: osdmap e47: 7 total, 6 up, 7 in 2024-05-08T06:33:26.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:25 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-08T06:33:26.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:25 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-08T06:33:26.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:25 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:26.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:25 smithi082 ceph-mon[22957]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-08T06:33:26.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:26 smithi134 ceph-mon[25063]: purged_snaps scrub starts 2024-05-08T06:33:26.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:26 smithi134 ceph-mon[25063]: purged_snaps scrub ok 2024-05-08T06:33:26.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:26 smithi134 ceph-mon[25063]: osd.6 [v2:172.21.15.134:6816/513779403,v1:172.21.15.134:6817/513779403] boot 2024-05-08T06:33:26.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:26 smithi134 ceph-mon[25063]: osdmap e48: 7 total, 7 up, 7 in 2024-05-08T06:33:26.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:26 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-08T06:33:27.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:26 smithi082 ceph-mon[22957]: purged_snaps scrub starts 2024-05-08T06:33:27.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:26 smithi082 ceph-mon[22957]: purged_snaps scrub ok 2024-05-08T06:33:27.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:26 smithi082 ceph-mon[22957]: osd.6 [v2:172.21.15.134:6816/513779403,v1:172.21.15.134:6817/513779403] boot 2024-05-08T06:33:27.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:26 smithi082 ceph-mon[22957]: osdmap e48: 7 total, 7 up, 7 in 2024-05-08T06:33:27.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:26 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-08T06:33:27.582 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-05-08T06:33:27.607 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph orch daemon add osd smithi134:vg_nvme/lv_1 2024-05-08T06:33:27.919 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:27 smithi134 ceph-mon[25063]: osdmap e49: 7 total, 7 up, 7 in 2024-05-08T06:33:27.919 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:27 smithi134 ceph-mon[25063]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-08T06:33:28.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:27 smithi082 ceph-mon[22957]: osdmap e49: 7 total, 7 up, 7 in 2024-05-08T06:33:28.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:27 smithi082 ceph-mon[22957]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-08T06:33:29.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:28 smithi134 ceph-mon[25063]: osdmap e50: 7 total, 7 up, 7 in 2024-05-08T06:33:29.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:28 smithi082 ceph-mon[22957]: osdmap e50: 7 total, 7 up, 7 in 2024-05-08T06:33:29.966 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:29 smithi134 ceph-mon[25063]: osdmap e51: 7 total, 7 up, 7 in 2024-05-08T06:33:29.967 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:29 smithi134 ceph-mon[25063]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-08T06:33:30.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:29 smithi082 ceph-mon[22957]: osdmap e51: 7 total, 7 up, 7 in 2024-05-08T06:33:30.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:29 smithi082 ceph-mon[22957]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-08T06:33:31.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:30 smithi134 ceph-mon[25063]: osdmap e52: 7 total, 7 up, 7 in 2024-05-08T06:33:31.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:30 smithi134 ceph-mon[25063]: Detected new or changed devices on smithi134 2024-05-08T06:33:31.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:31.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:33:31.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:33:31.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:33:31.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:30 smithi134 ceph-mon[25063]: Adjusting osd_memory_target on smithi134 to 5312M 2024-05-08T06:33:31.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:31.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:31.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:33:31.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:33:31.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:33:31.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:31.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:30 smithi134 ceph-mon[25063]: from='client.24189 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:33:31.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-08T06:33:31.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-08T06:33:31.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:33:31.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:30 smithi082 ceph-mon[22957]: osdmap e52: 7 total, 7 up, 7 in 2024-05-08T06:33:31.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:30 smithi082 ceph-mon[22957]: Detected new or changed devices on smithi134 2024-05-08T06:33:31.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:31.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:33:31.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:33:31.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:33:31.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:30 smithi082 ceph-mon[22957]: Adjusting osd_memory_target on smithi134 to 5312M 2024-05-08T06:33:31.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:31.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:31.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:33:31.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:33:31.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:33:31.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:31.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:30 smithi082 ceph-mon[22957]: from='client.24189 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:33:31.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-08T06:33:31.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-08T06:33:31.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:33:32.010 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:31 smithi134 ceph-mon[25063]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-05-08T06:33:32.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:31 smithi082 ceph-mon[22957]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-05-08T06:33:33.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:33 smithi082 ceph-mon[22957]: from='client.? 172.21.15.134:0/1069248889' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a515d7aa-b5ef-4a92-9241-97b2a19fbf8f"}]: dispatch 2024-05-08T06:33:33.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:33 smithi082 ceph-mon[22957]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a515d7aa-b5ef-4a92-9241-97b2a19fbf8f"}]: dispatch 2024-05-08T06:33:33.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:33 smithi082 ceph-mon[22957]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a515d7aa-b5ef-4a92-9241-97b2a19fbf8f"}]': finished 2024-05-08T06:33:33.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:33 smithi082 ceph-mon[22957]: osdmap e53: 8 total, 7 up, 8 in 2024-05-08T06:33:33.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:33 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-08T06:33:33.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:33 smithi134 ceph-mon[25063]: from='client.? 172.21.15.134:0/1069248889' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a515d7aa-b5ef-4a92-9241-97b2a19fbf8f"}]: dispatch 2024-05-08T06:33:33.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:33 smithi134 ceph-mon[25063]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a515d7aa-b5ef-4a92-9241-97b2a19fbf8f"}]: dispatch 2024-05-08T06:33:33.529 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:33 smithi134 ceph-mon[25063]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a515d7aa-b5ef-4a92-9241-97b2a19fbf8f"}]': finished 2024-05-08T06:33:33.529 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:33 smithi134 ceph-mon[25063]: osdmap e53: 8 total, 7 up, 8 in 2024-05-08T06:33:33.529 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:33 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-08T06:33:34.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:34 smithi082 ceph-mon[22957]: from='client.? 172.21.15.134:0/2577794905' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-08T06:33:34.417 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:34 smithi082 ceph-mon[22957]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-05-08T06:33:34.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:34 smithi134 ceph-mon[25063]: from='client.? 172.21.15.134:0/2577794905' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-08T06:33:34.528 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:34 smithi134 ceph-mon[25063]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-05-08T06:33:36.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:36 smithi134 ceph-mon[25063]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-05-08T06:33:37.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:36 smithi082 ceph-mon[22957]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-05-08T06:33:37.910 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:33:37.910 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:33:38.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:33:38.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:33:38.747 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:38 smithi134 ceph-mon[25063]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-08T06:33:39.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:38 smithi082 ceph-mon[22957]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-08T06:33:40.005 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:39 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-08T06:33:40.005 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:39 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:33:40.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:39 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-08T06:33:40.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:39 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:33:41.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:40 smithi134 ceph-mon[25063]: Deploying daemon osd.7 on smithi134 2024-05-08T06:33:41.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:40 smithi134 ceph-mon[25063]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-08T06:33:41.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:40 smithi082 ceph-mon[22957]: Deploying daemon osd.7 on smithi134 2024-05-08T06:33:41.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:40 smithi082 ceph-mon[22957]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-08T06:33:41.999 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:41 smithi134 ceph-mon[25063]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-05-08T06:33:42.068 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:41 smithi082 ceph-mon[22957]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-05-08T06:33:44.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:44 smithi134 ceph-mon[25063]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-05-08T06:33:44.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:44 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:44.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:44 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:33:44.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:44 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:33:44.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:44 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:33:44.837 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:44 smithi082 ceph-mon[22957]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-05-08T06:33:44.838 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:44 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:44.838 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:44 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:33:44.838 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:44 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:33:44.838 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:44 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:33:47.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:46 smithi134 ceph-mon[25063]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-05-08T06:33:47.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:46 smithi082 ceph-mon[22957]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-05-08T06:33:47.793 INFO:teuthology.orchestra.run.smithi134.stdout:Created osd(s) 7 on host 'smithi134' 2024-05-08T06:33:48.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:47 smithi134 ceph-mon[25063]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-05-08T06:33:48.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:47 smithi134 ceph-mon[25063]: from='osd.7 [v2:172.21.15.134:6824/2413145594,v1:172.21.15.134:6825/2413145594]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-08T06:33:48.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:47 smithi134 ceph-mon[25063]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-08T06:33:48.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:47 smithi082 ceph-mon[22957]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-05-08T06:33:48.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:47 smithi082 ceph-mon[22957]: from='osd.7 [v2:172.21.15.134:6824/2413145594,v1:172.21.15.134:6825/2413145594]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-08T06:33:48.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:47 smithi082 ceph-mon[22957]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-08T06:33:48.473 DEBUG:teuthology.orchestra.run.smithi134:osd.7> sudo journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.7.service 2024-05-08T06:33:48.477 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-05-08T06:33:48.477 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph osd stat -f json 2024-05-08T06:33:48.791 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:48 smithi082 ceph-mon[22957]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-08T06:33:48.791 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:48 smithi082 ceph-mon[22957]: osdmap e54: 8 total, 7 up, 8 in 2024-05-08T06:33:48.791 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:48 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-08T06:33:48.791 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:48 smithi082 ceph-mon[22957]: from='osd.7 [v2:172.21.15.134:6824/2413145594,v1:172.21.15.134:6825/2413145594]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-05-08T06:33:48.792 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:48 smithi082 ceph-mon[22957]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-05-08T06:33:48.792 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:48 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:48.975 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:48 smithi134 ceph-mon[25063]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-08T06:33:48.976 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:48 smithi134 ceph-mon[25063]: osdmap e54: 8 total, 7 up, 8 in 2024-05-08T06:33:48.976 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:48 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-08T06:33:48.976 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:48 smithi134 ceph-mon[25063]: from='osd.7 [v2:172.21.15.134:6824/2413145594,v1:172.21.15.134:6825/2413145594]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-05-08T06:33:48.976 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:48 smithi134 ceph-mon[25063]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-05-08T06:33:48.976 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:48 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:48.976 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:33:48 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:33:48.714+0000 7f4658245700 -1 osd.7 0 waiting for initial osdmap 2024-05-08T06:33:48.976 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:33:48 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:33:48.721+0000 7f4650bd8700 -1 osd.7 55 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-08T06:33:49.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:49 smithi134 ceph-mon[25063]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-05-08T06:33:49.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:49 smithi134 ceph-mon[25063]: osdmap e55: 8 total, 7 up, 8 in 2024-05-08T06:33:49.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:49 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-08T06:33:49.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:49 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-08T06:33:49.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:49 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:49.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:49 smithi134 ceph-mon[25063]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-05-08T06:33:50.054 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:49 smithi082 ceph-mon[22957]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-05-08T06:33:50.054 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:49 smithi082 ceph-mon[22957]: osdmap e55: 8 total, 7 up, 8 in 2024-05-08T06:33:50.054 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:49 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-08T06:33:50.054 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:49 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-08T06:33:50.055 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:49 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:50.055 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:49 smithi082 ceph-mon[22957]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-05-08T06:33:50.542 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:33:51.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:50 smithi134 ceph-mon[25063]: purged_snaps scrub starts 2024-05-08T06:33:51.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:50 smithi134 ceph-mon[25063]: purged_snaps scrub ok 2024-05-08T06:33:51.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:50 smithi134 ceph-mon[25063]: osd.7 [v2:172.21.15.134:6824/2413145594,v1:172.21.15.134:6825/2413145594] boot 2024-05-08T06:33:51.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:50 smithi134 ceph-mon[25063]: osdmap e56: 8 total, 8 up, 8 in 2024-05-08T06:33:51.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:50 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-08T06:33:51.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:50 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/18274620' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-08T06:33:51.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:50 smithi134 ceph-mon[25063]: osdmap e57: 8 total, 8 up, 8 in 2024-05-08T06:33:51.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:50 smithi082 ceph-mon[22957]: purged_snaps scrub starts 2024-05-08T06:33:51.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:50 smithi082 ceph-mon[22957]: purged_snaps scrub ok 2024-05-08T06:33:51.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:50 smithi082 ceph-mon[22957]: osd.7 [v2:172.21.15.134:6824/2413145594,v1:172.21.15.134:6825/2413145594] boot 2024-05-08T06:33:51.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:50 smithi082 ceph-mon[22957]: osdmap e56: 8 total, 8 up, 8 in 2024-05-08T06:33:51.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:50 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-08T06:33:51.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:50 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/18274620' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-08T06:33:51.168 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:50 smithi082 ceph-mon[22957]: osdmap e57: 8 total, 8 up, 8 in 2024-05-08T06:33:51.276 INFO:teuthology.orchestra.run.smithi082.stdout:{"epoch":56,"num_osds":8,"num_up_osds":8,"osd_up_since":1715150029,"num_in_osds":8,"osd_in_since":1715150013,"num_remapped_pgs":1} 2024-05-08T06:33:51.277 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph osd dump --format=json 2024-05-08T06:33:51.517 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:33:52.957 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:52 smithi082 ceph-mon[22957]: pgmap v172: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:33:52.958 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:52 smithi082 ceph-mon[22957]: osdmap e58: 8 total, 8 up, 8 in 2024-05-08T06:33:53.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:52 smithi134 ceph-mon[25063]: pgmap v172: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:33:53.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:52 smithi134 ceph-mon[25063]: osdmap e58: 8 total, 8 up, 8 in 2024-05-08T06:33:53.354 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:33:53.354 INFO:teuthology.orchestra.run.smithi082.stdout:{"epoch":59,"fsid":"02181fb0-0d04-11ef-bc97-c7b262605968","created":"2024-05-08T06:27:55.742942+0000","modified":"2024-05-08T06:33:52.679193+0000","last_up_change":"2024-05-08T06:33:49.712019+0000","last_in_change":"2024-05-08T06:33:33.001344+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-08T06:31:47.915658+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"b40392a7-c031-463b-9b63-5841f6f0ef6c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6802","nonce":3933757761},{"type":"v1","addr":"172.21.15.82:6803","nonce":3933757761}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6804","nonce":3933757761},{"type":"v1","addr":"172.21.15.82:6805","nonce":3933757761}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6808","nonce":3933757761},{"type":"v1","addr":"172.21.15.82:6809","nonce":3933757761}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6806","nonce":3933757761},{"type":"v1","addr":"172.21.15.82:6807","nonce":3933757761}]},"public_addr":"172.21.15.82:6803/3933757761","cluster_addr":"172.21.15.82:6805/3933757761","heartbeat_back_addr":"172.21.15.82:6809/3933757761","heartbeat_front_addr":"172.21.15.82:6807/3933757761","state":["exists","up"]},{"osd":1,"uuid":"e647dd10-311f-4617-8671-b32f4c79b571","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6810","nonce":2786774225},{"type":"v1","addr":"172.21.15.82:6811","nonce":2786774225}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6812","nonce":2786774225},{"type":"v1","addr":"172.21.15.82:6813","nonce":2786774225}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6816","nonce":2786774225},{"type":"v1","addr":"172.21.15.82:6817","nonce":2786774225}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6814","nonce":2786774225},{"type":"v1","addr":"172.21.15.82:6815","nonce":2786774225}]},"public_addr":"172.21.15.82:6811/2786774225","cluster_addr":"172.21.15.82:6813/2786774225","heartbeat_back_addr":"172.21.15.82:6817/2786774225","heartbeat_front_addr":"172.21.15.82:6815/2786774225","state":["exists","up"]},{"osd":2,"uuid":"f0eb5a47-398a-4541-b9b4-3743c790eb31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6818","nonce":2782041648},{"type":"v1","addr":"172.21.15.82:6819","nonce":2782041648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6820","nonce":2782041648},{"type":"v1","addr":"172.21.15.82:6821","nonce":2782041648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6824","nonce":2782041648},{"type":"v1","addr":"172.21.15.82:6825","nonce":2782041648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6822","nonce":2782041648},{"type":"v1","addr":"172.21.15.82:6823","nonce":2782041648}]},"public_addr":"172.21.15.82:6819/2782041648","cluster_addr":"172.21.15.82:6821/2782041648","heartbeat_back_addr":"172.21.15.82:6825/2782041648","heartbeat_front_addr":"172.21.15.82:6823/2782041648","state":["exists","up"]},{"osd":3,"uuid":"b0ebf498-c074-4bc5-97f1-780affa07b5d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6826","nonce":461300794},{"type":"v1","addr":"172.21.15.82:6827","nonce":461300794}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6828","nonce":461300794},{"type":"v1","addr":"172.21.15.82:6829","nonce":461300794}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6832","nonce":461300794},{"type":"v1","addr":"172.21.15.82:6833","nonce":461300794}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6830","nonce":461300794},{"type":"v1","addr":"172.21.15.82:6831","nonce":461300794}]},"public_addr":"172.21.15.82:6827/461300794","cluster_addr":"172.21.15.82:6829/461300794","heartbeat_back_addr":"172.21.15.82:6833/461300794","heartbeat_front_addr":"172.21.15.82:6831/461300794","state":["exists","up"]},{"osd":4,"uuid":"fa3c8174-2305-4238-8db5-2e328a64b9b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":2685273941},{"type":"v1","addr":"172.21.15.134:6801","nonce":2685273941}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":2685273941},{"type":"v1","addr":"172.21.15.134:6803","nonce":2685273941}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":2685273941},{"type":"v1","addr":"172.21.15.134:6807","nonce":2685273941}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":2685273941},{"type":"v1","addr":"172.21.15.134:6805","nonce":2685273941}]},"public_addr":"172.21.15.134:6801/2685273941","cluster_addr":"172.21.15.134:6803/2685273941","heartbeat_back_addr":"172.21.15.134:6807/2685273941","heartbeat_front_addr":"172.21.15.134:6805/2685273941","state":["exists","up"]},{"osd":5,"uuid":"37d8326a-b22d-41a3-aed4-5999fd2a0d09","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1347883706},{"type":"v1","addr":"172.21.15.134:6809","nonce":1347883706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1347883706},{"type":"v1","addr":"172.21.15.134:6811","nonce":1347883706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1347883706},{"type":"v1","addr":"172.21.15.134:6815","nonce":1347883706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1347883706},{"type":"v1","addr":"172.21.15.134:6813","nonce":1347883706}]},"public_addr":"172.21.15.134:6809/1347883706","cluster_addr":"172.21.15.134:6811/1347883706","heartbeat_back_addr":"172.21.15.134:6815/1347883706","heartbeat_front_addr":"172.21.15.134:6813/1347883706","state":["exists","up"]},{"osd":6,"uuid":"67965443-21f8-49ce-817e-27360f3cd976","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":513779403},{"type":"v1","addr":"172.21.15.134:6817","nonce":513779403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":513779403},{"type":"v1","addr":"172.21.15.134:6819","nonce":513779403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":513779403},{"type":"v1","addr":"172.21.15.134:6823","nonce":513779403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":513779403},{"type":"v1","addr":"172.21.15.134:6821","nonce":513779403}]},"public_addr":"172.21.15.134:6817/513779403","cluster_addr":"172.21.15.134:6819/513779403","heartbeat_back_addr":"172.21.15.134:6823/513779403","heartbeat_front_addr":"172.21.15.134:6821/513779403","state":["exists","up"]},{"osd":7,"uuid":"a515d7aa-b5ef-4a92-9241-97b2a19fbf8f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":2413145594},{"type":"v1","addr":"172.21.15.134:6825","nonce":2413145594}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":2413145594},{"type":"v1","addr":"172.21.15.134:6827","nonce":2413145594}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":2413145594},{"type":"v1","addr":"172.21.15.134:6831","nonce":2413145594}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":2413145594},{"type":"v1","addr":"172.21.15.134:6829","nonce":2413145594}]},"public_addr":"172.21.15.134:6825/2413145594","cluster_addr":"172.21.15.134:6827/2413145594","heartbeat_back_addr":"172.21.15.134:6831/2413145594","heartbeat_front_addr":"172.21.15.134:6829/2413145594","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:30:52.879155+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:31:17.748178+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:31:45.024027+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:32:12.314029+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:32:34.630511+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:32:59.157624+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:33:23.845075+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:33:48.679423+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.82:6801/4293212589":"2024-05-09T06:29:37.550605+0000","172.21.15.82:0/3421894798":"2024-05-09T06:29:37.550605+0000","172.21.15.82:0/3998155279":"2024-05-09T06:29:37.550605+0000","172.21.15.82:0/2791729147":"2024-05-09T06:28:48.192591+0000","172.21.15.82:6800/4088142804":"2024-05-09T06:28:48.192591+0000","172.21.15.82:0/1159340184":"2024-05-09T06:28:48.192591+0000","172.21.15.82:0/410482785":"2024-05-09T06:28:48.192591+0000","172.21.15.82:0/2906783232":"2024-05-09T06:28:18.800220+0000","172.21.15.82:0/3551893116":"2024-05-09T06:28:18.800220+0000","172.21.15.82:6800/4293212589":"2024-05-09T06:29:37.550605+0000","172.21.15.82:6801/4088142804":"2024-05-09T06:28:48.192591+0000","172.21.15.82:6800/910282116":"2024-05-09T06:28:18.800220+0000","172.21.15.82:6801/910282116":"2024-05-09T06:28:18.800220+0000","172.21.15.82:0/562327267":"2024-05-09T06:29:37.550605+0000","172.21.15.82:0/3376450769":"2024-05-09T06:28:18.800220+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-08T06:33:54.015 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:53 smithi082 ceph-mon[22957]: osdmap e59: 8 total, 8 up, 8 in 2024-05-08T06:33:54.015 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:53 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:54.015 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:53 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:33:54.015 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:53 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:33:54.015 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:53 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:33:54.015 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:53 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:33:54.015 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:53 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:54.015 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:53 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:54.015 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:53 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:33:54.016 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:53 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:33:54.016 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:53 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:33:54.016 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:53 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:54.016 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:53 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/4205458785' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-08T06:33:54.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:53 smithi134 ceph-mon[25063]: osdmap e59: 8 total, 8 up, 8 in 2024-05-08T06:33:54.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:53 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:54.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:53 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:33:54.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:53 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:33:54.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:53 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:33:54.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:53 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-08T06:33:54.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:53 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:54.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:53 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:54.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:53 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:33:54.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:53 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:33:54.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:53 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:33:54.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:53 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:33:54.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:53 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/4205458785' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-08T06:33:54.039 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-05-08T06:31:47.915658+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-05-08T06:33:54.040 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-05-08T06:33:54.254 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:33:55.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:54 smithi134 ceph-mon[25063]: Detected new or changed devices on smithi134 2024-05-08T06:33:55.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:54 smithi134 ceph-mon[25063]: Adjusting osd_memory_target on smithi134 to 3984M 2024-05-08T06:33:55.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:54 smithi134 ceph-mon[25063]: pgmap v175: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:33:55.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:54 smithi134 ceph-mon[25063]: osdmap e60: 8 total, 8 up, 8 in 2024-05-08T06:33:55.073 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:54 smithi082 ceph-mon[22957]: Detected new or changed devices on smithi134 2024-05-08T06:33:55.073 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:54 smithi082 ceph-mon[22957]: Adjusting osd_memory_target on smithi134 to 3984M 2024-05-08T06:33:55.073 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:54 smithi082 ceph-mon[22957]: pgmap v175: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:33:55.073 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:54 smithi082 ceph-mon[22957]: osdmap e60: 8 total, 8 up, 8 in 2024-05-08T06:33:56.192 INFO:teuthology.orchestra.run.smithi082.stdout:pg_num: 1 2024-05-08T06:33:56.873 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:56 smithi082 ceph-mon[22957]: pgmap v177: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:33:56.873 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:56 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3319461904' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-08T06:33:56.897 INFO:tasks.cephadm:Setting up client nodes... 2024-05-08T06:33:56.897 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-05-08T06:33:57.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:56 smithi134 ceph-mon[25063]: pgmap v177: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:33:57.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:56 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/3319461904' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-08T06:33:58.910 INFO:teuthology.orchestra.run.smithi082.stdout:[client.0] 2024-05-08T06:33:58.910 INFO:teuthology.orchestra.run.smithi082.stdout: key = AQDWHDtmck4ZNhAAgKztuOhEr6omi5TuUaE86g== 2024-05-08T06:33:58.910 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:58 smithi082 ceph-mon[22957]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-08T06:33:59.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:58 smithi134 ceph-mon[25063]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-08T06:33:59.559 DEBUG:teuthology.orchestra.run.smithi082:> set -ex 2024-05-08T06:33:59.559 DEBUG:teuthology.orchestra.run.smithi082:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-05-08T06:33:59.559 DEBUG:teuthology.orchestra.run.smithi082:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-05-08T06:33:59.598 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-05-08T06:33:59.598 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-05-08T06:33:59.598 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph mgr dump --format=json 2024-05-08T06:33:59.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:59 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/4032835213' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-08T06:33:59.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:33:59 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/4032835213' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-08T06:33:59.851 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:33:59.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:59 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/4032835213' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-08T06:33:59.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:33:59 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/4032835213' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-08T06:34:00.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:00 smithi082 ceph-mon[22957]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-08T06:34:01.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:00 smithi134 ceph-mon[25063]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-08T06:34:01.635 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:34:01.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:01 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3984518967' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-08T06:34:02.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:01 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/3984518967' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-08T06:34:02.414 INFO:teuthology.orchestra.run.smithi082.stdout:{"epoch":19,"active_gid":14200,"active_name":"smithi082.bsnvtb","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6800","nonce":3161062320},{"type":"v1","addr":"172.21.15.82:6801","nonce":3161062320}]},"active_addr":"172.21.15.82:6801/3161062320","active_change":"2024-05-08T06:29:37.550915+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14220,"name":"smithi134.zbsjta","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.82:8443/","prometheus":"http://172.21.15.82:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.82:0","nonce":3514796531}]},{"addrvec":[{"type":"v2","addr":"172.21.15.82:0","nonce":442017506}]},{"addrvec":[{"type":"v2","addr":"172.21.15.82:0","nonce":39972019}]},{"addrvec":[{"type":"v2","addr":"172.21.15.82:0","nonce":2154700551}]}]}} 2024-05-08T06:34:02.420 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-05-08T06:34:02.420 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-05-08T06:34:02.420 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph osd dump --format=json 2024-05-08T06:34:02.643 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:03.027 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:02 smithi082 ceph-mon[22957]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-08T06:34:03.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:02 smithi134 ceph-mon[25063]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-05-08T06:34:03.853 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:03 smithi082 ceph-mon[22957]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-08T06:34:04.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:03 smithi134 ceph-mon[25063]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-08T06:34:04.245 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:34:04.245 INFO:teuthology.orchestra.run.smithi082.stdout:{"epoch":60,"fsid":"02181fb0-0d04-11ef-bc97-c7b262605968","created":"2024-05-08T06:27:55.742942+0000","modified":"2024-05-08T06:33:53.683402+0000","last_up_change":"2024-05-08T06:33:49.712019+0000","last_in_change":"2024-05-08T06:33:33.001344+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-08T06:31:47.915658+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"b40392a7-c031-463b-9b63-5841f6f0ef6c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6802","nonce":3933757761},{"type":"v1","addr":"172.21.15.82:6803","nonce":3933757761}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6804","nonce":3933757761},{"type":"v1","addr":"172.21.15.82:6805","nonce":3933757761}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6808","nonce":3933757761},{"type":"v1","addr":"172.21.15.82:6809","nonce":3933757761}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6806","nonce":3933757761},{"type":"v1","addr":"172.21.15.82:6807","nonce":3933757761}]},"public_addr":"172.21.15.82:6803/3933757761","cluster_addr":"172.21.15.82:6805/3933757761","heartbeat_back_addr":"172.21.15.82:6809/3933757761","heartbeat_front_addr":"172.21.15.82:6807/3933757761","state":["exists","up"]},{"osd":1,"uuid":"e647dd10-311f-4617-8671-b32f4c79b571","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6810","nonce":2786774225},{"type":"v1","addr":"172.21.15.82:6811","nonce":2786774225}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6812","nonce":2786774225},{"type":"v1","addr":"172.21.15.82:6813","nonce":2786774225}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6816","nonce":2786774225},{"type":"v1","addr":"172.21.15.82:6817","nonce":2786774225}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6814","nonce":2786774225},{"type":"v1","addr":"172.21.15.82:6815","nonce":2786774225}]},"public_addr":"172.21.15.82:6811/2786774225","cluster_addr":"172.21.15.82:6813/2786774225","heartbeat_back_addr":"172.21.15.82:6817/2786774225","heartbeat_front_addr":"172.21.15.82:6815/2786774225","state":["exists","up"]},{"osd":2,"uuid":"f0eb5a47-398a-4541-b9b4-3743c790eb31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6818","nonce":2782041648},{"type":"v1","addr":"172.21.15.82:6819","nonce":2782041648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6820","nonce":2782041648},{"type":"v1","addr":"172.21.15.82:6821","nonce":2782041648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6824","nonce":2782041648},{"type":"v1","addr":"172.21.15.82:6825","nonce":2782041648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6822","nonce":2782041648},{"type":"v1","addr":"172.21.15.82:6823","nonce":2782041648}]},"public_addr":"172.21.15.82:6819/2782041648","cluster_addr":"172.21.15.82:6821/2782041648","heartbeat_back_addr":"172.21.15.82:6825/2782041648","heartbeat_front_addr":"172.21.15.82:6823/2782041648","state":["exists","up"]},{"osd":3,"uuid":"b0ebf498-c074-4bc5-97f1-780affa07b5d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6826","nonce":461300794},{"type":"v1","addr":"172.21.15.82:6827","nonce":461300794}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6828","nonce":461300794},{"type":"v1","addr":"172.21.15.82:6829","nonce":461300794}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6832","nonce":461300794},{"type":"v1","addr":"172.21.15.82:6833","nonce":461300794}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6830","nonce":461300794},{"type":"v1","addr":"172.21.15.82:6831","nonce":461300794}]},"public_addr":"172.21.15.82:6827/461300794","cluster_addr":"172.21.15.82:6829/461300794","heartbeat_back_addr":"172.21.15.82:6833/461300794","heartbeat_front_addr":"172.21.15.82:6831/461300794","state":["exists","up"]},{"osd":4,"uuid":"fa3c8174-2305-4238-8db5-2e328a64b9b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":2685273941},{"type":"v1","addr":"172.21.15.134:6801","nonce":2685273941}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":2685273941},{"type":"v1","addr":"172.21.15.134:6803","nonce":2685273941}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":2685273941},{"type":"v1","addr":"172.21.15.134:6807","nonce":2685273941}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":2685273941},{"type":"v1","addr":"172.21.15.134:6805","nonce":2685273941}]},"public_addr":"172.21.15.134:6801/2685273941","cluster_addr":"172.21.15.134:6803/2685273941","heartbeat_back_addr":"172.21.15.134:6807/2685273941","heartbeat_front_addr":"172.21.15.134:6805/2685273941","state":["exists","up"]},{"osd":5,"uuid":"37d8326a-b22d-41a3-aed4-5999fd2a0d09","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1347883706},{"type":"v1","addr":"172.21.15.134:6809","nonce":1347883706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1347883706},{"type":"v1","addr":"172.21.15.134:6811","nonce":1347883706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1347883706},{"type":"v1","addr":"172.21.15.134:6815","nonce":1347883706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1347883706},{"type":"v1","addr":"172.21.15.134:6813","nonce":1347883706}]},"public_addr":"172.21.15.134:6809/1347883706","cluster_addr":"172.21.15.134:6811/1347883706","heartbeat_back_addr":"172.21.15.134:6815/1347883706","heartbeat_front_addr":"172.21.15.134:6813/1347883706","state":["exists","up"]},{"osd":6,"uuid":"67965443-21f8-49ce-817e-27360f3cd976","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":513779403},{"type":"v1","addr":"172.21.15.134:6817","nonce":513779403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":513779403},{"type":"v1","addr":"172.21.15.134:6819","nonce":513779403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":513779403},{"type":"v1","addr":"172.21.15.134:6823","nonce":513779403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":513779403},{"type":"v1","addr":"172.21.15.134:6821","nonce":513779403}]},"public_addr":"172.21.15.134:6817/513779403","cluster_addr":"172.21.15.134:6819/513779403","heartbeat_back_addr":"172.21.15.134:6823/513779403","heartbeat_front_addr":"172.21.15.134:6821/513779403","state":["exists","up"]},{"osd":7,"uuid":"a515d7aa-b5ef-4a92-9241-97b2a19fbf8f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":2413145594},{"type":"v1","addr":"172.21.15.134:6825","nonce":2413145594}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":2413145594},{"type":"v1","addr":"172.21.15.134:6827","nonce":2413145594}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":2413145594},{"type":"v1","addr":"172.21.15.134:6831","nonce":2413145594}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":2413145594},{"type":"v1","addr":"172.21.15.134:6829","nonce":2413145594}]},"public_addr":"172.21.15.134:6825/2413145594","cluster_addr":"172.21.15.134:6827/2413145594","heartbeat_back_addr":"172.21.15.134:6831/2413145594","heartbeat_front_addr":"172.21.15.134:6829/2413145594","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:30:52.879155+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:31:17.748178+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:31:45.024027+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:32:12.314029+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:32:34.630511+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:32:59.157624+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:33:23.845075+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:33:48.679423+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.82:6801/4293212589":"2024-05-09T06:29:37.550605+0000","172.21.15.82:0/3421894798":"2024-05-09T06:29:37.550605+0000","172.21.15.82:0/3998155279":"2024-05-09T06:29:37.550605+0000","172.21.15.82:0/2791729147":"2024-05-09T06:28:48.192591+0000","172.21.15.82:6800/4088142804":"2024-05-09T06:28:48.192591+0000","172.21.15.82:0/1159340184":"2024-05-09T06:28:48.192591+0000","172.21.15.82:0/410482785":"2024-05-09T06:28:48.192591+0000","172.21.15.82:0/2906783232":"2024-05-09T06:28:18.800220+0000","172.21.15.82:0/3551893116":"2024-05-09T06:28:18.800220+0000","172.21.15.82:6800/4293212589":"2024-05-09T06:29:37.550605+0000","172.21.15.82:6801/4088142804":"2024-05-09T06:28:48.192591+0000","172.21.15.82:6800/910282116":"2024-05-09T06:28:18.800220+0000","172.21.15.82:6801/910282116":"2024-05-09T06:28:18.800220+0000","172.21.15.82:0/562327267":"2024-05-09T06:29:37.550605+0000","172.21.15.82:0/3376450769":"2024-05-09T06:28:18.800220+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-08T06:34:04.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:04 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3529779647' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-08T06:34:04.971 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-05-08T06:34:04.971 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph osd dump --format=json 2024-05-08T06:34:05.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:04 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/3529779647' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-08T06:34:05.184 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:06.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:06 smithi082 ceph-mon[22957]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-05-08T06:34:07.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:06 smithi134 ceph-mon[25063]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-05-08T06:34:07.042 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:34:07.042 INFO:teuthology.orchestra.run.smithi082.stdout:{"epoch":60,"fsid":"02181fb0-0d04-11ef-bc97-c7b262605968","created":"2024-05-08T06:27:55.742942+0000","modified":"2024-05-08T06:33:53.683402+0000","last_up_change":"2024-05-08T06:33:49.712019+0000","last_in_change":"2024-05-08T06:33:33.001344+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-08T06:31:47.915658+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"b40392a7-c031-463b-9b63-5841f6f0ef6c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6802","nonce":3933757761},{"type":"v1","addr":"172.21.15.82:6803","nonce":3933757761}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6804","nonce":3933757761},{"type":"v1","addr":"172.21.15.82:6805","nonce":3933757761}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6808","nonce":3933757761},{"type":"v1","addr":"172.21.15.82:6809","nonce":3933757761}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6806","nonce":3933757761},{"type":"v1","addr":"172.21.15.82:6807","nonce":3933757761}]},"public_addr":"172.21.15.82:6803/3933757761","cluster_addr":"172.21.15.82:6805/3933757761","heartbeat_back_addr":"172.21.15.82:6809/3933757761","heartbeat_front_addr":"172.21.15.82:6807/3933757761","state":["exists","up"]},{"osd":1,"uuid":"e647dd10-311f-4617-8671-b32f4c79b571","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6810","nonce":2786774225},{"type":"v1","addr":"172.21.15.82:6811","nonce":2786774225}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6812","nonce":2786774225},{"type":"v1","addr":"172.21.15.82:6813","nonce":2786774225}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6816","nonce":2786774225},{"type":"v1","addr":"172.21.15.82:6817","nonce":2786774225}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6814","nonce":2786774225},{"type":"v1","addr":"172.21.15.82:6815","nonce":2786774225}]},"public_addr":"172.21.15.82:6811/2786774225","cluster_addr":"172.21.15.82:6813/2786774225","heartbeat_back_addr":"172.21.15.82:6817/2786774225","heartbeat_front_addr":"172.21.15.82:6815/2786774225","state":["exists","up"]},{"osd":2,"uuid":"f0eb5a47-398a-4541-b9b4-3743c790eb31","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6818","nonce":2782041648},{"type":"v1","addr":"172.21.15.82:6819","nonce":2782041648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6820","nonce":2782041648},{"type":"v1","addr":"172.21.15.82:6821","nonce":2782041648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6824","nonce":2782041648},{"type":"v1","addr":"172.21.15.82:6825","nonce":2782041648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6822","nonce":2782041648},{"type":"v1","addr":"172.21.15.82:6823","nonce":2782041648}]},"public_addr":"172.21.15.82:6819/2782041648","cluster_addr":"172.21.15.82:6821/2782041648","heartbeat_back_addr":"172.21.15.82:6825/2782041648","heartbeat_front_addr":"172.21.15.82:6823/2782041648","state":["exists","up"]},{"osd":3,"uuid":"b0ebf498-c074-4bc5-97f1-780affa07b5d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6826","nonce":461300794},{"type":"v1","addr":"172.21.15.82:6827","nonce":461300794}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6828","nonce":461300794},{"type":"v1","addr":"172.21.15.82:6829","nonce":461300794}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6832","nonce":461300794},{"type":"v1","addr":"172.21.15.82:6833","nonce":461300794}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.82:6830","nonce":461300794},{"type":"v1","addr":"172.21.15.82:6831","nonce":461300794}]},"public_addr":"172.21.15.82:6827/461300794","cluster_addr":"172.21.15.82:6829/461300794","heartbeat_back_addr":"172.21.15.82:6833/461300794","heartbeat_front_addr":"172.21.15.82:6831/461300794","state":["exists","up"]},{"osd":4,"uuid":"fa3c8174-2305-4238-8db5-2e328a64b9b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":2685273941},{"type":"v1","addr":"172.21.15.134:6801","nonce":2685273941}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":2685273941},{"type":"v1","addr":"172.21.15.134:6803","nonce":2685273941}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":2685273941},{"type":"v1","addr":"172.21.15.134:6807","nonce":2685273941}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":2685273941},{"type":"v1","addr":"172.21.15.134:6805","nonce":2685273941}]},"public_addr":"172.21.15.134:6801/2685273941","cluster_addr":"172.21.15.134:6803/2685273941","heartbeat_back_addr":"172.21.15.134:6807/2685273941","heartbeat_front_addr":"172.21.15.134:6805/2685273941","state":["exists","up"]},{"osd":5,"uuid":"37d8326a-b22d-41a3-aed4-5999fd2a0d09","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1347883706},{"type":"v1","addr":"172.21.15.134:6809","nonce":1347883706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1347883706},{"type":"v1","addr":"172.21.15.134:6811","nonce":1347883706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1347883706},{"type":"v1","addr":"172.21.15.134:6815","nonce":1347883706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1347883706},{"type":"v1","addr":"172.21.15.134:6813","nonce":1347883706}]},"public_addr":"172.21.15.134:6809/1347883706","cluster_addr":"172.21.15.134:6811/1347883706","heartbeat_back_addr":"172.21.15.134:6815/1347883706","heartbeat_front_addr":"172.21.15.134:6813/1347883706","state":["exists","up"]},{"osd":6,"uuid":"67965443-21f8-49ce-817e-27360f3cd976","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":513779403},{"type":"v1","addr":"172.21.15.134:6817","nonce":513779403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":513779403},{"type":"v1","addr":"172.21.15.134:6819","nonce":513779403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":513779403},{"type":"v1","addr":"172.21.15.134:6823","nonce":513779403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":513779403},{"type":"v1","addr":"172.21.15.134:6821","nonce":513779403}]},"public_addr":"172.21.15.134:6817/513779403","cluster_addr":"172.21.15.134:6819/513779403","heartbeat_back_addr":"172.21.15.134:6823/513779403","heartbeat_front_addr":"172.21.15.134:6821/513779403","state":["exists","up"]},{"osd":7,"uuid":"a515d7aa-b5ef-4a92-9241-97b2a19fbf8f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":2413145594},{"type":"v1","addr":"172.21.15.134:6825","nonce":2413145594}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":2413145594},{"type":"v1","addr":"172.21.15.134:6827","nonce":2413145594}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":2413145594},{"type":"v1","addr":"172.21.15.134:6831","nonce":2413145594}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":2413145594},{"type":"v1","addr":"172.21.15.134:6829","nonce":2413145594}]},"public_addr":"172.21.15.134:6825/2413145594","cluster_addr":"172.21.15.134:6827/2413145594","heartbeat_back_addr":"172.21.15.134:6831/2413145594","heartbeat_front_addr":"172.21.15.134:6829/2413145594","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:30:52.879155+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:31:17.748178+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:31:45.024027+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:32:12.314029+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:32:34.630511+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:32:59.157624+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:33:23.845075+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-08T06:33:48.679423+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.82:6801/4293212589":"2024-05-09T06:29:37.550605+0000","172.21.15.82:0/3421894798":"2024-05-09T06:29:37.550605+0000","172.21.15.82:0/3998155279":"2024-05-09T06:29:37.550605+0000","172.21.15.82:0/2791729147":"2024-05-09T06:28:48.192591+0000","172.21.15.82:6800/4088142804":"2024-05-09T06:28:48.192591+0000","172.21.15.82:0/1159340184":"2024-05-09T06:28:48.192591+0000","172.21.15.82:0/410482785":"2024-05-09T06:28:48.192591+0000","172.21.15.82:0/2906783232":"2024-05-09T06:28:18.800220+0000","172.21.15.82:0/3551893116":"2024-05-09T06:28:18.800220+0000","172.21.15.82:6800/4293212589":"2024-05-09T06:29:37.550605+0000","172.21.15.82:6801/4088142804":"2024-05-09T06:28:48.192591+0000","172.21.15.82:6800/910282116":"2024-05-09T06:28:18.800220+0000","172.21.15.82:6801/910282116":"2024-05-09T06:28:18.800220+0000","172.21.15.82:0/562327267":"2024-05-09T06:29:37.550605+0000","172.21.15.82:0/3376450769":"2024-05-09T06:28:18.800220+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-08T06:34:07.687 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-05-08T06:34:07.687 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-05-08T06:34:07.688 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-05-08T06:34:07.688 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-05-08T06:34:07.688 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-05-08T06:34:07.689 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-05-08T06:34:07.689 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-05-08T06:34:07.690 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-05-08T06:34:07.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:07 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3271262714' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-08T06:34:08.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:07 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/3271262714' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-08T06:34:08.162 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:08.164 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:08.165 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:08.168 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:08.169 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:08.169 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:08.169 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:08.172 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:08.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:08 smithi082 ceph-mon[22957]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-05-08T06:34:09.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:08 smithi134 ceph-mon[25063]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-05-08T06:34:10.951 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:10 smithi082 ceph-mon[22957]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:11.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:10 smithi134 ceph-mon[25063]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:13.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:12 smithi134 ceph-mon[25063]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:13.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:12 smithi082 ceph-mon[22957]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:14.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:14 smithi082 ceph-mon[22957]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:15.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:14 smithi134 ceph-mon[25063]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:16.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:16 smithi082 ceph-mon[22957]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:17.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:16 smithi134 ceph-mon[25063]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:18.836 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:18 smithi082 ceph-mon[22957]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:19.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:18 smithi134 ceph-mon[25063]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:20.303 INFO:teuthology.orchestra.run.smithi082.stdout:206158430220 2024-05-08T06:34:20.304 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-05-08T06:34:20.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:20 smithi082 ceph-mon[22957]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:20.972 INFO:teuthology.orchestra.run.smithi082.stdout:171798691857 2024-05-08T06:34:20.973 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-05-08T06:34:21.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:20 smithi134 ceph-mon[25063]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:21.559 INFO:teuthology.orchestra.run.smithi082.stdout:38654705707 2024-05-08T06:34:21.559 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-05-08T06:34:21.961 INFO:teuthology.orchestra.run.smithi082.stdout:81604378658 2024-05-08T06:34:21.962 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-05-08T06:34:22.904 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:22 smithi082 ceph-mon[22957]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:22.920 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:22.928 INFO:teuthology.orchestra.run.smithi082.stdout:240518168585 2024-05-08T06:34:22.928 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-05-08T06:34:22.946 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:23.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:22 smithi134 ceph-mon[25063]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:23.338 INFO:teuthology.orchestra.run.smithi082.stdout:137438953495 2024-05-08T06:34:23.338 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-05-08T06:34:23.580 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:23.581 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:23.938 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:23 smithi082 ceph-mon[22957]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:23.968 INFO:teuthology.orchestra.run.smithi082.stdout:115964117021 2024-05-08T06:34:23.968 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-05-08T06:34:24.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:23 smithi134 ceph-mon[25063]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:24.680 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:27.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:26 smithi134 ceph-mon[25063]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:27.068 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:26 smithi082 ceph-mon[22957]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:28.825 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:28 smithi082 ceph-mon[22957]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:28.894 INFO:teuthology.orchestra.run.smithi082.stdout:81604378659 2024-05-08T06:34:29.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:28 smithi134 ceph-mon[25063]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:29.561 INFO:teuthology.orchestra.run.smithi082.stdout:38654705709 2024-05-08T06:34:29.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:29 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/836811231' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-08T06:34:29.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:29 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/4180318998' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-08T06:34:29.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:29 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/836811231' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-08T06:34:29.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:29 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/4180318998' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-08T06:34:30.029 INFO:teuthology.orchestra.run.smithi082.stdout:60129542184 2024-05-08T06:34:30.029 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-05-08T06:34:30.744 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:30 smithi082 ceph-mon[22957]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:30.744 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:30 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/1126664338' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-08T06:34:30.767 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705707 got 38654705709 for osd.0 2024-05-08T06:34:30.767 DEBUG:teuthology.parallel:result is None 2024-05-08T06:34:31.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:30 smithi134 ceph-mon[25063]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:31.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:30 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/1126664338' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-08T06:34:31.515 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378658 got 81604378659 for osd.2 2024-05-08T06:34:31.515 DEBUG:teuthology.parallel:result is None 2024-05-08T06:34:31.749 INFO:teuthology.orchestra.run.smithi082.stdout:171798691860 2024-05-08T06:34:31.943 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:31.944 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:31.944 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:32.843 INFO:teuthology.orchestra.run.smithi082.stdout:240518168588 2024-05-08T06:34:32.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:32 smithi082 ceph-mon[22957]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:33.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:32 smithi134 ceph-mon[25063]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:33.421 INFO:teuthology.orchestra.run.smithi082.stdout:206158430223 2024-05-08T06:34:34.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:33 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/3517087685' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-08T06:34:34.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:33 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/845833967' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-08T06:34:34.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:33 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3517087685' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-08T06:34:34.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:33 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/845833967' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-08T06:34:34.492 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691857 got 171798691860 for osd.5 2024-05-08T06:34:34.493 DEBUG:teuthology.parallel:result is None 2024-05-08T06:34:34.886 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:34 smithi082 ceph-mon[22957]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:34.920 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168585 got 240518168588 for osd.7 2024-05-08T06:34:34.920 DEBUG:teuthology.parallel:result is None 2024-05-08T06:34:35.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:34 smithi134 ceph-mon[25063]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:35.276 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430220 got 206158430223 for osd.6 2024-05-08T06:34:35.276 DEBUG:teuthology.parallel:result is None 2024-05-08T06:34:36.908 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:36 smithi082 ceph-mon[22957]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:37.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:36 smithi134 ceph-mon[25063]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:37.817 INFO:teuthology.orchestra.run.smithi082.stdout:60129542187 2024-05-08T06:34:37.990 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:34:37.990 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:34:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:34:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:34:38.777 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542184 got 60129542187 for osd.1 2024-05-08T06:34:38.777 DEBUG:teuthology.parallel:result is None 2024-05-08T06:34:38.972 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:38 smithi082 ceph-mon[22957]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:38.972 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:38 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/802514427' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-08T06:34:39.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:38 smithi134 ceph-mon[25063]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:39.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:38 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/802514427' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-08T06:34:39.169 INFO:teuthology.orchestra.run.smithi082.stdout:115964117024 2024-05-08T06:34:39.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:39 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/765406591' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-08T06:34:39.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:39 smithi134 ceph-mon[25063]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:39.822 INFO:teuthology.orchestra.run.smithi082.stdout:137438953499 2024-05-08T06:34:39.962 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:39 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/765406591' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-08T06:34:39.962 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:39 smithi082 ceph-mon[22957]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:39.984 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117021 got 115964117024 for osd.3 2024-05-08T06:34:39.984 DEBUG:teuthology.parallel:result is None 2024-05-08T06:34:40.573 INFO:tasks.cephadm.ceph_manager.ceph:need seq 137438953495 got 137438953499 for osd.4 2024-05-08T06:34:40.574 DEBUG:teuthology.parallel:result is None 2024-05-08T06:34:40.574 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-05-08T06:34:40.574 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph pg dump --format=json 2024-05-08T06:34:40.791 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:40.914 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:40 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3357078479' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-08T06:34:41.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:40 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/3357078479' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-08T06:34:41.838 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:41 smithi082 ceph-mon[22957]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:42.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:41 smithi134 ceph-mon[25063]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:42.468 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:34:42.470 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2024-05-08T06:34:43.027 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:42 smithi082 ceph-mon[22957]: from='client.14448 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-08T06:34:43.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:42 smithi134 ceph-mon[25063]: from='client.14448 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-08T06:34:43.070 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":200,"stamp":"2024-05-08T06:34:41.657950+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":49072,"kb_used_data":6384,"kb_used_omap":0,"kb_used_meta":42624,"kb_avail":749912144,"statfs":{"total":767960285184,"available":767910035456,"internally_reserved":0,"allocated":6537216,"data_stored":3571298,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43646976},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002122"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":223,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-05-08T06:33:53.696425+0000","last_change":"2024-05-08T06:33:53.696425+0000","last_active":"2024-05-08T06:33:53.696425+0000","last_peered":"2024-05-08T06:33:53.696425+0000","last_clean":"2024-05-08T06:33:53.696425+0000","last_became_active":"2024-05-08T06:33:53.695935+0000","last_became_peered":"2024-05-08T06:33:53.695935+0000","last_unstale":"2024-05-08T06:33:53.696425+0000","last_undegraded":"2024-05-08T06:33:53.696425+0000","last_fullsized":"2024-05-08T06:33:53.696425+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-08T06:31:48.668453+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-08T06:31:48.668453+0000","last_clean_scrub_stamp":"2024-05-08T06:31:48.668453+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-09T11:43:40.800299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2662400,"data_stored":2644640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168590,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6060,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739092,"statfs":{"total":95995035648,"available":95988830208,"internally_reserved":0,"allocated":1150976,"data_stored":777150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57699999999999996}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60999999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.627}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51800000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36899999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40300000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45300000000000001}]}]},{"osd":6,"up_from":48,"seq":206158430225,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6188,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738964,"statfs":{"total":95995035648,"available":95988699136,"internally_reserved":0,"allocated":1150976,"data_stored":777150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 8 06:34:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.53300000000000003}]},{"osd":1,"last update":"Wed May 8 06:34:26 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":8.9670000000000005,"5min":8.9670000000000005,"15min":8.9670000000000005},"last":0.50700000000000001}]},{"osd":2,"last update":"Wed May 8 06:34:26 2024","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":8.9450000000000003,"5min":8.9450000000000003,"15min":8.9450000000000003},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.38800000000000001}]},{"osd":3,"last update":"Wed May 8 06:34:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.55100000000000005}]},{"osd":4,"last update":"Wed May 8 06:34:26 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.48099999999999998}]},{"osd":5,"last update":"Wed May 8 06:34:26 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.499},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.56899999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58199999999999996}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6368,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738784,"statfs":{"total":95995035648,"available":95988514816,"internally_reserved":0,"allocated":614400,"data_stored":247591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 8 06:34:30 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.34999999999999998,"15min":0.318},"min":{"1min":0.26600000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":1.794,"5min":1.794,"15min":1.794},"last":0.312},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.36199999999999999,"15min":0.316},"min":{"1min":0.29699999999999999,"5min":0.186,"15min":0.186},"max":{"1min":1.716,"5min":1.716,"15min":1.716},"last":0.28499999999999998}]},{"osd":2,"last update":"Wed May 8 06:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.35599999999999998,"15min":0.34300000000000003},"min":{"1min":0.28899999999999998,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.38200000000000001,"15min":0.36799999999999999},"min":{"1min":0.35599999999999998,"5min":0.222,"15min":0.222},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.27200000000000002}]},{"osd":3,"last update":"Wed May 8 06:34:18 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.442,"15min":0.44400000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.48299999999999998,"15min":0.48899999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.3400000000000001,"5min":1.3400000000000001,"15min":1.3400000000000001},"last":0.30299999999999999}]},{"osd":4,"last update":"Wed May 8 06:34:38 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.58299999999999996,"15min":0.57999999999999996},"min":{"1min":0.38900000000000001,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.4830000000000001,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.436}]},{"osd":5,"last update":"Wed May 8 06:34:02 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.47399999999999998}]},{"osd":6,"last update":"Wed May 8 06:34:30 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.609,"5min":1.609,"15min":1.609},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.405,"5min":1.405,"15min":1.405},"last":0.44700000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40400000000000003}]}]},{"osd":0,"up_from":9,"seq":38654705712,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed May 8 06:34:25 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.34799999999999998,"15min":0.32200000000000001},"min":{"1min":0.21199999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.53900000000000003,"5min":0.62,"15min":0.62},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.372,"5min":0.36099999999999999,"15min":0.33300000000000002},"min":{"1min":0.26200000000000001,"5min":0.221,"15min":0.221},"max":{"1min":0.54100000000000004,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.26300000000000001}]},{"osd":2,"last update":"Wed May 8 06:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.39800000000000002,"15min":0.39400000000000002},"min":{"1min":0.308,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.55800000000000005,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.42999999999999999,"15min":0.42599999999999999},"min":{"1min":0.28599999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.60499999999999998,"5min":0.626,"15min":0.626},"last":0.39000000000000001}]},{"osd":3,"last update":"Wed May 8 06:34:18 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.49199999999999999,"15min":0.497},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.46500000000000002,"15min":0.47499999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.64100000000000001,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.312}]},{"osd":4,"last update":"Wed May 8 06:34:38 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.63,"15min":0.63100000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.88,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.60799999999999998,"15min":0.61399999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.73399999999999999,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.67700000000000005}]},{"osd":5,"last update":"Wed May 8 06:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.82399999999999995}]},{"osd":6,"last update":"Wed May 8 06:34:26 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.84799999999999998},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.70099999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64300000000000002}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 8 06:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.36599999999999999,"15min":0.36099999999999999},"min":{"1min":0.28000000000000003,"5min":0.252,"15min":0.252},"max":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.374,"15min":0.36499999999999999},"min":{"1min":0.29999999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.29199999999999998}]},{"osd":1,"last update":"Wed May 8 06:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.38,"15min":0.36799999999999999},"min":{"1min":0.27700000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.39300000000000002,"15min":0.38300000000000001},"min":{"1min":0.32400000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.249}]},{"osd":3,"last update":"Wed May 8 06:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.44500000000000001,"15min":0.441},"min":{"1min":0.33700000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.307},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.46100000000000002,"15min":0.45900000000000002},"min":{"1min":0.35899999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.35199999999999998}]},{"osd":4,"last update":"Wed May 8 06:33:38 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.751},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.54400000000000004}]},{"osd":5,"last update":"Wed May 8 06:34:01 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.72399999999999998}]},{"osd":6,"last update":"Wed May 8 06:34:26 2024","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.67200000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73199999999999998}]}]},{"osd":3,"up_from":27,"seq":115964117025,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 8 06:34:14 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.39800000000000002,"15min":0.40100000000000002},"min":{"1min":0.223,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.59899999999999998,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41899999999999998,"15min":0.41999999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.56299999999999994,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.497}]},{"osd":1,"last update":"Wed May 8 06:34:14 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.44,"15min":0.44600000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.57499999999999996,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.42899999999999999,"15min":0.433},"min":{"1min":0.25800000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.55600000000000005,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.51600000000000001}]},{"osd":2,"last update":"Wed May 8 06:34:14 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.45000000000000001,"15min":0.45100000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.433,"5min":0.442,"15min":0.44400000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.63800000000000001}]},{"osd":4,"last update":"Wed May 8 06:33:39 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.54700000000000004}]},{"osd":5,"last update":"Wed May 8 06:34:01 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.68100000000000005}]},{"osd":6,"last update":"Wed May 8 06:34:27 2024","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.69199999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72699999999999998}]}]},{"osd":4,"up_from":32,"seq":137438953500,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5796,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739356,"statfs":{"total":95995035648,"available":95989100544,"internally_reserved":0,"allocated":618496,"data_stored":248222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 8 06:34:39 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.53200000000000003,"15min":0.53500000000000003},"min":{"1min":0.35899999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.71699999999999997,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.57399999999999995,"15min":0.57999999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.72099999999999997,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.495}]},{"osd":1,"last update":"Wed May 8 06:34:39 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.53700000000000003,"15min":0.54500000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.66700000000000004,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.55300000000000005,"15min":0.55600000000000005},"min":{"1min":0.39400000000000002,"5min":0.372,"15min":0.372},"max":{"1min":0.70999999999999996,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.47399999999999998}]},{"osd":2,"last update":"Wed May 8 06:34:39 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.54700000000000004,"15min":0.54500000000000004},"min":{"1min":0.41699999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.72899999999999998,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.68700000000000006,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.41199999999999998}]},{"osd":3,"last update":"Wed May 8 06:34:39 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.57499999999999996,"15min":0.57899999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.70099999999999996,"5min":0.877,"15min":0.877},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.58699999999999997,"15min":0.59299999999999997},"min":{"1min":0.44,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.69999999999999996,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.44}]},{"osd":5,"last update":"Wed May 8 06:34:03 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.51900000000000002}]},{"osd":6,"last update":"Wed May 8 06:34:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.57899999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56000000000000005}]}]},{"osd":5,"up_from":40,"seq":171798691863,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5728,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739424,"statfs":{"total":95995035648,"available":95989170176,"internally_reserved":0,"allocated":614400,"data_stored":247591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 8 06:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.46300000000000002}]},{"osd":1,"last update":"Wed May 8 06:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.51400000000000001}]},{"osd":2,"last update":"Wed May 8 06:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.54200000000000004}]},{"osd":3,"last update":"Wed May 8 06:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.499},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.59399999999999997}]},{"osd":4,"last update":"Wed May 8 06:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.58299999999999996}]},{"osd":6,"last update":"Wed May 8 06:34:28 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.61199999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57399999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-08T06:34:43.072 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph pg dump --format=json 2024-05-08T06:34:43.299 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:43.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:43 smithi082 ceph-mon[22957]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:44.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:43 smithi134 ceph-mon[25063]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:45.019 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:34:45.021 INFO:teuthology.orchestra.run.smithi082.stderr:dumped all 2024-05-08T06:34:45.624 INFO:teuthology.orchestra.run.smithi082.stdout:{"pg_ready":true,"pg_map":{"version":201,"stamp":"2024-05-08T06:34:43.658235+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":49072,"kb_used_data":6384,"kb_used_omap":0,"kb_used_meta":42624,"kb_avail":749912144,"statfs":{"total":767960285184,"available":767910035456,"internally_reserved":0,"allocated":6537216,"data_stored":3571298,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43646976},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002114"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":223,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-05-08T06:33:53.696425+0000","last_change":"2024-05-08T06:33:53.696425+0000","last_active":"2024-05-08T06:33:53.696425+0000","last_peered":"2024-05-08T06:33:53.696425+0000","last_clean":"2024-05-08T06:33:53.696425+0000","last_became_active":"2024-05-08T06:33:53.695935+0000","last_became_peered":"2024-05-08T06:33:53.695935+0000","last_unstale":"2024-05-08T06:33:53.696425+0000","last_undegraded":"2024-05-08T06:33:53.696425+0000","last_fullsized":"2024-05-08T06:33:53.696425+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-08T06:31:48.668453+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-08T06:31:48.668453+0000","last_clean_scrub_stamp":"2024-05-08T06:31:48.668453+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-09T11:43:40.800299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2662400,"data_stored":2644640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168590,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6060,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739092,"statfs":{"total":95995035648,"available":95988830208,"internally_reserved":0,"allocated":1150976,"data_stored":777150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57699999999999996}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60999999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.627}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51800000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36899999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40300000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45300000000000001}]}]},{"osd":6,"up_from":48,"seq":206158430225,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6188,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738964,"statfs":{"total":95995035648,"available":95988699136,"internally_reserved":0,"allocated":1150976,"data_stored":777150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 8 06:34:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.53300000000000003}]},{"osd":1,"last update":"Wed May 8 06:34:26 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":8.9670000000000005,"5min":8.9670000000000005,"15min":8.9670000000000005},"last":0.50700000000000001}]},{"osd":2,"last update":"Wed May 8 06:34:26 2024","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":8.9450000000000003,"5min":8.9450000000000003,"15min":8.9450000000000003},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.38800000000000001}]},{"osd":3,"last update":"Wed May 8 06:34:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.55100000000000005}]},{"osd":4,"last update":"Wed May 8 06:34:26 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.48099999999999998}]},{"osd":5,"last update":"Wed May 8 06:34:26 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.499},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.56899999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58199999999999996}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6368,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738784,"statfs":{"total":95995035648,"available":95988514816,"internally_reserved":0,"allocated":614400,"data_stored":247591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 8 06:34:30 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.34999999999999998,"15min":0.318},"min":{"1min":0.26600000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":1.794,"5min":1.794,"15min":1.794},"last":0.312},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.36199999999999999,"15min":0.316},"min":{"1min":0.29699999999999999,"5min":0.186,"15min":0.186},"max":{"1min":1.716,"5min":1.716,"15min":1.716},"last":0.28499999999999998}]},{"osd":2,"last update":"Wed May 8 06:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.35599999999999998,"15min":0.34300000000000003},"min":{"1min":0.28899999999999998,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.38200000000000001,"15min":0.36799999999999999},"min":{"1min":0.35599999999999998,"5min":0.222,"15min":0.222},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.27200000000000002}]},{"osd":3,"last update":"Wed May 8 06:34:18 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.442,"15min":0.44400000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.48299999999999998,"15min":0.48899999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.3400000000000001,"5min":1.3400000000000001,"15min":1.3400000000000001},"last":0.30299999999999999}]},{"osd":4,"last update":"Wed May 8 06:34:38 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.58299999999999996,"15min":0.57999999999999996},"min":{"1min":0.38900000000000001,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.4830000000000001,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.436}]},{"osd":5,"last update":"Wed May 8 06:34:02 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.47399999999999998}]},{"osd":6,"last update":"Wed May 8 06:34:30 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.609,"5min":1.609,"15min":1.609},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.405,"5min":1.405,"15min":1.405},"last":0.44700000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40400000000000003}]}]},{"osd":0,"up_from":9,"seq":38654705712,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed May 8 06:34:25 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.34799999999999998,"15min":0.32200000000000001},"min":{"1min":0.21199999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.53900000000000003,"5min":0.62,"15min":0.62},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.372,"5min":0.36099999999999999,"15min":0.33300000000000002},"min":{"1min":0.26200000000000001,"5min":0.221,"15min":0.221},"max":{"1min":0.54100000000000004,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.26300000000000001}]},{"osd":2,"last update":"Wed May 8 06:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.39800000000000002,"15min":0.39400000000000002},"min":{"1min":0.308,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.55800000000000005,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.42999999999999999,"15min":0.42599999999999999},"min":{"1min":0.28599999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.60499999999999998,"5min":0.626,"15min":0.626},"last":0.39000000000000001}]},{"osd":3,"last update":"Wed May 8 06:34:18 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.49199999999999999,"15min":0.497},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.46500000000000002,"15min":0.47499999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.64100000000000001,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.312}]},{"osd":4,"last update":"Wed May 8 06:34:38 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.63,"15min":0.63100000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.88,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.60799999999999998,"15min":0.61399999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.73399999999999999,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.67700000000000005}]},{"osd":5,"last update":"Wed May 8 06:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.82399999999999995}]},{"osd":6,"last update":"Wed May 8 06:34:26 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.84799999999999998},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.70099999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64300000000000002}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 8 06:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.36599999999999999,"15min":0.36099999999999999},"min":{"1min":0.28000000000000003,"5min":0.252,"15min":0.252},"max":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.374,"15min":0.36499999999999999},"min":{"1min":0.29999999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.28100000000000003}]},{"osd":1,"last update":"Wed May 8 06:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.38,"15min":0.36799999999999999},"min":{"1min":0.27700000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.25},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.39300000000000002,"15min":0.38300000000000001},"min":{"1min":0.32400000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.35999999999999999}]},{"osd":3,"last update":"Wed May 8 06:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.44500000000000001,"15min":0.441},"min":{"1min":0.33700000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.46100000000000002,"15min":0.45900000000000002},"min":{"1min":0.35899999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.30399999999999999}]},{"osd":4,"last update":"Wed May 8 06:34:38 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.58799999999999997,"15min":0.58599999999999997},"min":{"1min":0.41599999999999998,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.622,"5min":0.59499999999999997,"15min":0.59099999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.45300000000000001}]},{"osd":5,"last update":"Wed May 8 06:34:01 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.497}]},{"osd":6,"last update":"Wed May 8 06:34:26 2024","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.46899999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51400000000000001}]}]},{"osd":3,"up_from":27,"seq":115964117025,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 8 06:34:14 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.39800000000000002,"15min":0.40100000000000002},"min":{"1min":0.223,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.59899999999999998,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41899999999999998,"15min":0.41999999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.56299999999999994,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.497}]},{"osd":1,"last update":"Wed May 8 06:34:14 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.44,"15min":0.44600000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.57499999999999996,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.42899999999999999,"15min":0.433},"min":{"1min":0.25800000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.55600000000000005,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.51600000000000001}]},{"osd":2,"last update":"Wed May 8 06:34:14 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.45000000000000001,"15min":0.45100000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.433,"5min":0.442,"15min":0.44400000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.63800000000000001}]},{"osd":4,"last update":"Wed May 8 06:33:39 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.54700000000000004}]},{"osd":5,"last update":"Wed May 8 06:34:01 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.68100000000000005}]},{"osd":6,"last update":"Wed May 8 06:34:27 2024","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.69199999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72699999999999998}]}]},{"osd":4,"up_from":32,"seq":137438953500,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5796,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739356,"statfs":{"total":95995035648,"available":95989100544,"internally_reserved":0,"allocated":618496,"data_stored":248222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 8 06:34:39 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.53200000000000003,"15min":0.53500000000000003},"min":{"1min":0.35899999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.71699999999999997,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.57399999999999995,"15min":0.57999999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.72099999999999997,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.495}]},{"osd":1,"last update":"Wed May 8 06:34:39 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.53700000000000003,"15min":0.54500000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.66700000000000004,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.55300000000000005,"15min":0.55600000000000005},"min":{"1min":0.39400000000000002,"5min":0.372,"15min":0.372},"max":{"1min":0.70999999999999996,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.47399999999999998}]},{"osd":2,"last update":"Wed May 8 06:34:39 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.54700000000000004,"15min":0.54500000000000004},"min":{"1min":0.41699999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.72899999999999998,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.68700000000000006,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.41199999999999998}]},{"osd":3,"last update":"Wed May 8 06:34:39 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.57499999999999996,"15min":0.57899999999999996},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.70099999999999996,"5min":0.877,"15min":0.877},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.58699999999999997,"15min":0.59299999999999997},"min":{"1min":0.44,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.69999999999999996,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.44}]},{"osd":5,"last update":"Wed May 8 06:34:03 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.51900000000000002}]},{"osd":6,"last update":"Wed May 8 06:34:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.57899999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56000000000000005}]}]},{"osd":5,"up_from":40,"seq":171798691863,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5728,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739424,"statfs":{"total":95995035648,"available":95989170176,"internally_reserved":0,"allocated":614400,"data_stored":247591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed May 8 06:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.46300000000000002}]},{"osd":1,"last update":"Wed May 8 06:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.51400000000000001}]},{"osd":2,"last update":"Wed May 8 06:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.54200000000000004}]},{"osd":3,"last update":"Wed May 8 06:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.499},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.59399999999999997}]},{"osd":4,"last update":"Wed May 8 06:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.58299999999999996}]},{"osd":6,"last update":"Wed May 8 06:34:28 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.61199999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57399999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-08T06:34:45.625 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-05-08T06:34:45.625 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-05-08T06:34:45.625 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-05-08T06:34:45.626 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph health --format=json 2024-05-08T06:34:45.843 INFO:teuthology.orchestra.run.smithi082.stderr:Inferring config /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/mon.smithi082/config 2024-05-08T06:34:47.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:46 smithi134 ceph-mon[25063]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-08T06:34:47.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:46 smithi134 ceph-mon[25063]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:47.037 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:46 smithi082 ceph-mon[22957]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-08T06:34:47.037 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:46 smithi082 ceph-mon[22957]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:47.543 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:34:47.543 INFO:teuthology.orchestra.run.smithi082.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-05-08T06:34:48.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:47 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/3124508890' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-08T06:34:48.094 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:47 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3124508890' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-08T06:34:48.120 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-05-08T06:34:48.121 INFO:tasks.cephadm:Setup complete, yielding 2024-05-08T06:34:48.121 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-08T06:34:48.130 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi082.front.sepia.ceph.com 2024-05-08T06:34:48.131 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- bash -c 'ceph orch status' 2024-05-08T06:34:49.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:48 smithi134 ceph-mon[25063]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:49.103 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:48 smithi082 ceph-mon[22957]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:50.022 INFO:teuthology.orchestra.run.smithi082.stdout:Backend: cephadm 2024-05-08T06:34:50.023 INFO:teuthology.orchestra.run.smithi082.stdout:Available: Yes 2024-05-08T06:34:50.023 INFO:teuthology.orchestra.run.smithi082.stdout:Paused: No 2024-05-08T06:34:50.592 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- bash -c 'ceph orch ps' 2024-05-08T06:34:50.869 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:50 smithi082 ceph-mon[22957]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:51.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:50 smithi134 ceph-mon[25063]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:51.965 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:51 smithi082 ceph-mon[22957]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:34:52.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:51 smithi134 ceph-mon[25063]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:34:52.375 INFO:teuthology.orchestra.run.smithi082.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-08T06:34:52.375 INFO:teuthology.orchestra.run.smithi082.stdout:alertmanager.smithi082 smithi082 *:9093,9094 running (4m) 2m ago 5m 17.6M - ba2b418f427c 86af446dfdba 2024-05-08T06:34:52.375 INFO:teuthology.orchestra.run.smithi082.stdout:crash.smithi082 smithi082 running (5m) 2m ago 5m 6409k - 17.2.0 e1d6a67b021e c5efbe70843e 2024-05-08T06:34:52.375 INFO:teuthology.orchestra.run.smithi082.stdout:crash.smithi134 smithi134 running (5m) 62s ago 5m 6405k - 17.2.0 e1d6a67b021e c670ee9715d5 2024-05-08T06:34:52.375 INFO:teuthology.orchestra.run.smithi082.stdout:grafana.smithi082 smithi082 *:3000 running (3m) 2m ago 5m 45.2M - 8.3.5 dad864ee21e9 903d0b5aab5f 2024-05-08T06:34:52.375 INFO:teuthology.orchestra.run.smithi082.stdout:mgr.smithi082.bsnvtb smithi082 *:9283 running (6m) 2m ago 6m 448M - 17.2.0 e1d6a67b021e affb5f53b21e 2024-05-08T06:34:52.375 INFO:teuthology.orchestra.run.smithi082.stdout:mgr.smithi134.zbsjta smithi134 *:8443,9283 running (4m) 62s ago 4m 412M - 17.2.0 e1d6a67b021e 971c6830c247 2024-05-08T06:34:52.375 INFO:teuthology.orchestra.run.smithi082.stdout:mon.smithi082 smithi082 running (6m) 2m ago 6m 41.3M 2048M 17.2.0 e1d6a67b021e ee82d4cf1e0f 2024-05-08T06:34:52.375 INFO:teuthology.orchestra.run.smithi082.stdout:mon.smithi134 smithi134 running (4m) 62s ago 4m 41.9M 2048M 17.2.0 e1d6a67b021e 302cfd0d2bb4 2024-05-08T06:34:52.375 INFO:teuthology.orchestra.run.smithi082.stdout:node-exporter.smithi082 smithi082 *:9100 running (5m) 2m ago 5m 15.2M - 1dbe0e931976 3ba57430c975 2024-05-08T06:34:52.376 INFO:teuthology.orchestra.run.smithi082.stdout:node-exporter.smithi134 smithi134 *:9100 running (4m) 62s ago 4m 16.5M - 1dbe0e931976 aea98e24f85a 2024-05-08T06:34:52.376 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0 smithi082 running (4m) 2m ago 4m 35.9M 3216M 17.2.0 e1d6a67b021e 4061f89d770d 2024-05-08T06:34:52.376 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1 smithi082 running (3m) 2m ago 3m 37.6M 3216M 17.2.0 e1d6a67b021e 2c86afbd7dc1 2024-05-08T06:34:52.376 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2 smithi082 running (3m) 2m ago 3m 32.0M 3216M 17.2.0 e1d6a67b021e a703fa1e14f7 2024-05-08T06:34:52.376 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3 smithi082 running (2m) 2m ago 2m 25.9M 3216M 17.2.0 e1d6a67b021e 7f3b8f015f59 2024-05-08T06:34:52.376 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4 smithi134 running (2m) 62s ago 2m 34.8M 3984M 17.2.0 e1d6a67b021e 3de640027e4e 2024-05-08T06:34:52.376 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5 smithi134 running (117s) 62s ago 117s 37.8M 3984M 17.2.0 e1d6a67b021e 75f363c21b07 2024-05-08T06:34:52.376 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6 smithi134 running (93s) 62s ago 92s 30.4M 3984M 17.2.0 e1d6a67b021e 98bcf9455cd6 2024-05-08T06:34:52.376 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7 smithi134 running (68s) 62s ago 67s 10.8M 3984M 17.2.0 e1d6a67b021e 9b1f288b1e8c 2024-05-08T06:34:52.376 INFO:teuthology.orchestra.run.smithi082.stdout:prometheus.smithi082 smithi082 *:9095 running (4m) 2m ago 4m 54.7M - 514e6a882f6e 637ca038131b 2024-05-08T06:34:52.965 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:52 smithi082 ceph-mon[22957]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:52.989 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- bash -c 'ceph orch ls' 2024-05-08T06:34:53.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:52 smithi134 ceph-mon[25063]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:53.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:53 smithi082 ceph-mon[22957]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:34:53.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:53 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:34:53.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:53 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:34:53.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:53 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:34:53.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:53 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:34:54.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:53 smithi134 ceph-mon[25063]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:34:54.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:53 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:34:54.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:53 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:34:54.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:53 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:34:54.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:53 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:34:54.898 INFO:teuthology.orchestra.run.smithi082.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-05-08T06:34:54.898 INFO:teuthology.orchestra.run.smithi082.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-05-08T06:34:54.898 INFO:teuthology.orchestra.run.smithi082.stdout:crash 2/2 2m ago 6m * 2024-05-08T06:34:54.898 INFO:teuthology.orchestra.run.smithi082.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-05-08T06:34:54.898 INFO:teuthology.orchestra.run.smithi082.stdout:mgr 2/2 2m ago 6m count:2 2024-05-08T06:34:54.898 INFO:teuthology.orchestra.run.smithi082.stdout:mon 2/2 2m ago 5m smithi082:172.21.15.82=smithi082;smithi134:172.21.15.134=smithi134;count:2 2024-05-08T06:34:54.899 INFO:teuthology.orchestra.run.smithi082.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-05-08T06:34:54.899 INFO:teuthology.orchestra.run.smithi082.stdout:osd 8 2m ago - 2024-05-08T06:34:54.899 INFO:teuthology.orchestra.run.smithi082.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-05-08T06:34:54.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:54 smithi082 ceph-mon[22957]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:55.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:54 smithi134 ceph-mon[25063]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:55.469 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- bash -c 'ceph orch host ls' 2024-05-08T06:34:56.943 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:56 smithi082 ceph-mon[22957]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:34:56.943 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:56 smithi082 ceph-mon[22957]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:57.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:56 smithi134 ceph-mon[25063]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:34:57.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:56 smithi134 ceph-mon[25063]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:57.323 INFO:teuthology.orchestra.run.smithi082.stdout:HOST ADDR LABELS STATUS 2024-05-08T06:34:57.323 INFO:teuthology.orchestra.run.smithi082.stdout:smithi082 172.21.15.82 2024-05-08T06:34:57.323 INFO:teuthology.orchestra.run.smithi082.stdout:smithi134 172.21.15.134 2024-05-08T06:34:57.323 INFO:teuthology.orchestra.run.smithi082.stdout:2 hosts in cluster 2024-05-08T06:34:57.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:57 smithi082 ceph-mon[22957]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:34:57.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:57 smithi082 ceph-mon[22957]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:57.942 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- bash -c 'ceph orch device ls' 2024-05-08T06:34:58.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:57 smithi134 ceph-mon[25063]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:34:58.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:57 smithi134 ceph-mon[25063]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:34:59.722 INFO:teuthology.orchestra.run.smithi082.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-05-08T06:34:59.722 INFO:teuthology.orchestra.run.smithi082.stdout:smithi082 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400SD400BGN 400G LVM detected, locked 2024-05-08T06:34:59.722 INFO:teuthology.orchestra.run.smithi082.stdout:smithi134 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT62420002400BGN 400G LVM detected, locked 2024-05-08T06:35:00.021 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:34:59 smithi082 ceph-mon[22957]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:35:00.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:34:59 smithi134 ceph-mon[25063]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:35:00.305 INFO:teuthology.run_tasks:Running task vip.exec... 2024-05-08T06:35:00.335 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi082.front.sepia.ceph.com 2024-05-08T06:35:00.335 DEBUG:teuthology.orchestra.run.smithi082:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-05-08T06:35:00.361 INFO:teuthology.orchestra.run.smithi082.stderr:+ systemctl stop nfs-server 2024-05-08T06:35:00.368 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi134.front.sepia.ceph.com 2024-05-08T06:35:00.369 DEBUG:teuthology.orchestra.run.smithi134:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-05-08T06:35:00.394 INFO:teuthology.orchestra.run.smithi134.stderr:+ systemctl stop nfs-server 2024-05-08T06:35:00.402 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-08T06:35:00.411 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi082.front.sepia.ceph.com 2024-05-08T06:35:00.411 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-05-08T06:35:01.954 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:01 smithi082 ceph-mon[22957]: from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:02.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:01 smithi134 ceph-mon[25063]: from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:03.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:02 smithi134 ceph-mon[25063]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:35:03.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:02 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-05-08T06:35:03.049 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:02 smithi082 ceph-mon[22957]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:35:03.049 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:02 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-05-08T06:35:04.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:03 smithi134 ceph-mon[25063]: from='client.14480 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:04.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:03 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-05-08T06:35:04.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:03 smithi134 ceph-mon[25063]: osdmap e61: 8 total, 8 up, 8 in 2024-05-08T06:35:04.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:03 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-05-08T06:35:04.035 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:03 smithi082 ceph-mon[22957]: from='client.14480 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:04.035 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:03 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-05-08T06:35:04.035 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:03 smithi082 ceph-mon[22957]: osdmap e61: 8 total, 8 up, 8 in 2024-05-08T06:35:04.036 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:03 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-05-08T06:35:04.036 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:03 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi082[22952]: 2024-05-08T06:35:03.712+0000 7fa5f9450700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-05-08T06:35:04.274 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-05-08T06:35:04.284 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-05-08T06:35:04.284 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph orch ls -f json 2024-05-08T06:35:04.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:04 smithi082 ceph-mon[22957]: pgmap v212: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:35:04.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-05-08T06:35:04.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:04 smithi082 ceph-mon[22957]: osdmap e62: 8 total, 8 up, 8 in 2024-05-08T06:35:04.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-05-08T06:35:04.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:04 smithi082 ceph-mon[22957]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-05-08T06:35:04.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:04 smithi082 ceph-mon[22957]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-05-08T06:35:04.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:04 smithi082 ceph-mon[22957]: osdmap e63: 8 total, 8 up, 8 in 2024-05-08T06:35:04.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-05-08T06:35:04.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:04 smithi082 ceph-mon[22957]: fsmap foofs:0 2024-05-08T06:35:04.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:04.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:35:04.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:04.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:35:04.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:04.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:04.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi082.ofnfgy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-08T06:35:04.919 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi082.ofnfgy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-08T06:35:04.919 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:05.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:04 smithi134 ceph-mon[25063]: pgmap v212: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:35:05.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-05-08T06:35:05.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:04 smithi134 ceph-mon[25063]: osdmap e62: 8 total, 8 up, 8 in 2024-05-08T06:35:05.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-05-08T06:35:05.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:04 smithi134 ceph-mon[25063]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-05-08T06:35:05.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:04 smithi134 ceph-mon[25063]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-05-08T06:35:05.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:04 smithi134 ceph-mon[25063]: osdmap e63: 8 total, 8 up, 8 in 2024-05-08T06:35:05.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-05-08T06:35:05.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:04 smithi134 ceph-mon[25063]: fsmap foofs:0 2024-05-08T06:35:05.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:05.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:35:05.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:05.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:35:05.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:05.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:05.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi082.ofnfgy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-08T06:35:05.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi082.ofnfgy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-08T06:35:05.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:05.796 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:05 smithi082 ceph-mon[22957]: Saving service mds.foofs spec with placement count:2 2024-05-08T06:35:05.796 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:05 smithi082 ceph-mon[22957]: Deploying daemon mds.foofs.smithi082.ofnfgy on smithi082 2024-05-08T06:35:05.796 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:05 smithi082 ceph-mon[22957]: osdmap e64: 8 total, 8 up, 8 in 2024-05-08T06:35:06.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:05 smithi134 ceph-mon[25063]: Saving service mds.foofs spec with placement count:2 2024-05-08T06:35:06.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:05 smithi134 ceph-mon[25063]: Deploying daemon mds.foofs.smithi082.ofnfgy on smithi082 2024-05-08T06:35:06.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:05 smithi134 ceph-mon[25063]: osdmap e64: 8 total, 8 up, 8 in 2024-05-08T06:35:07.012 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:06 smithi082 ceph-mon[22957]: pgmap v216: 65 pgs: 55 unknown, 10 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:35:07.012 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:06 smithi082 ceph-mon[22957]: osdmap e65: 8 total, 8 up, 8 in 2024-05-08T06:35:07.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:06 smithi134 ceph-mon[25063]: pgmap v216: 65 pgs: 55 unknown, 10 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:35:07.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:06 smithi134 ceph-mon[25063]: osdmap e65: 8 total, 8 up, 8 in 2024-05-08T06:35:07.092 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:35:07.092 INFO:teuthology.orchestra.run.smithi082.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-05-08T06:28:37.520432Z", "last_refresh": "2024-05-08T06:32:21.990821Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-05-08T06:28:31.461121Z", "last_refresh": "2024-05-08T06:32:21.990885Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-05-08T06:28:34.343205Z", "last_refresh": "2024-05-08T06:32:21.990977Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-05-08T06:35:03.733050Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-05-08T06:35:03.728027Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-05-08T06:28:29.448705Z", "last_refresh": "2024-05-08T06:32:21.990751Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi082:172.21.15.82=smithi082", "smithi134:172.21.15.134=smithi134"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-05-08T06:29:21.871524Z", "last_refresh": "2024-05-08T06:32:21.990611Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-05-08T06:28:35.536726Z", "last_refresh": "2024-05-08T06:32:21.991041Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-05-08T06:32:21.991166Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-05-08T06:28:32.598691Z", "last_refresh": "2024-05-08T06:32:21.991103Z", "ports": [9095], "running": 1, "size": 1}}] 2024-05-08T06:35:07.801 INFO:tasks.cephadm:mds.foofs has 0/2 2024-05-08T06:35:08.685 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:08 smithi134 ceph-mon[25063]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-08T06:35:08.685 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:08 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:08.685 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:08 smithi134 ceph-mon[25063]: pgmap v218: 65 pgs: 4 unknown, 61 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:35:08.685 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:08 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:08.686 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:08 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi134.ooovgp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-08T06:35:08.686 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:08 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi134.ooovgp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-08T06:35:08.686 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:08 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:08.802 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph orch ls -f json 2024-05-08T06:35:08.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:08 smithi082 ceph-mon[22957]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-08T06:35:08.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:08 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:08.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:08 smithi082 ceph-mon[22957]: pgmap v218: 65 pgs: 4 unknown, 61 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:35:08.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:08 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:08.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:08 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi134.ooovgp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-08T06:35:08.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:08 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi134.ooovgp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-08T06:35:08.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:08 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:09.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:09 smithi134 ceph-mon[25063]: Deploying daemon mds.foofs.smithi134.ooovgp on smithi134 2024-05-08T06:35:09.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:09 smithi134 ceph-mon[25063]: mds.? [v2:172.21.15.82:6834/2448674594,v1:172.21.15.82:6835/2448674594] up:boot 2024-05-08T06:35:09.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:09 smithi134 ceph-mon[25063]: daemon mds.foofs.smithi082.ofnfgy assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-05-08T06:35:09.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:09 smithi134 ceph-mon[25063]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-05-08T06:35:09.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:09 smithi134 ceph-mon[25063]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-05-08T06:35:09.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:09 smithi134 ceph-mon[25063]: Cluster is now healthy 2024-05-08T06:35:09.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:09 smithi134 ceph-mon[25063]: fsmap foofs:0 1 up:standby 2024-05-08T06:35:09.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:09 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi082.ofnfgy"}]: dispatch 2024-05-08T06:35:09.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:09 smithi134 ceph-mon[25063]: fsmap foofs:1 {0=foofs.smithi082.ofnfgy=up:creating} 2024-05-08T06:35:09.779 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:09 smithi134 ceph-mon[25063]: daemon mds.foofs.smithi082.ofnfgy is now active in filesystem foofs as rank 0 2024-05-08T06:35:09.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:09 smithi082 ceph-mon[22957]: Deploying daemon mds.foofs.smithi134.ooovgp on smithi134 2024-05-08T06:35:09.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:09 smithi082 ceph-mon[22957]: mds.? [v2:172.21.15.82:6834/2448674594,v1:172.21.15.82:6835/2448674594] up:boot 2024-05-08T06:35:09.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:09 smithi082 ceph-mon[22957]: daemon mds.foofs.smithi082.ofnfgy assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-05-08T06:35:09.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:09 smithi082 ceph-mon[22957]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-05-08T06:35:09.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:09 smithi082 ceph-mon[22957]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-05-08T06:35:09.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:09 smithi082 ceph-mon[22957]: Cluster is now healthy 2024-05-08T06:35:09.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:09 smithi082 ceph-mon[22957]: fsmap foofs:0 1 up:standby 2024-05-08T06:35:09.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:09 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi082.ofnfgy"}]: dispatch 2024-05-08T06:35:09.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:09 smithi082 ceph-mon[22957]: fsmap foofs:1 {0=foofs.smithi082.ofnfgy=up:creating} 2024-05-08T06:35:09.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:09 smithi082 ceph-mon[22957]: daemon mds.foofs.smithi082.ofnfgy is now active in filesystem foofs as rank 0 2024-05-08T06:35:10.764 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:35:10.764 INFO:teuthology.orchestra.run.smithi082.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-05-08T06:28:37.520432Z", "last_refresh": "2024-05-08T06:32:21.990821Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-05-08T06:28:31.461121Z", "last_refresh": "2024-05-08T06:32:21.990885Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-05-08T06:28:34.343205Z", "last_refresh": "2024-05-08T06:32:21.990977Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-05-08T06:35:03.733050Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-05-08T06:35:03.728027Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-05-08T06:28:29.448705Z", "last_refresh": "2024-05-08T06:32:21.990751Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi082:172.21.15.82=smithi082", "smithi134:172.21.15.134=smithi134"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-05-08T06:29:21.871524Z", "last_refresh": "2024-05-08T06:32:21.990611Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-05-08T06:28:35.536726Z", "last_refresh": "2024-05-08T06:32:21.991041Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-05-08T06:32:21.991166Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-05-08T06:28:32.598691Z", "last_refresh": "2024-05-08T06:32:21.991103Z", "ports": [9095], "running": 1, "size": 1}}] 2024-05-08T06:35:10.764 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:10 smithi082 ceph-mon[22957]: pgmap v219: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:35:10.764 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:10 smithi082 ceph-mon[22957]: mds.? [v2:172.21.15.82:6834/2448674594,v1:172.21.15.82:6835/2448674594] up:active 2024-05-08T06:35:10.764 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:10 smithi082 ceph-mon[22957]: fsmap foofs:1 {0=foofs.smithi082.ofnfgy=up:active} 2024-05-08T06:35:10.764 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:10 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:10.764 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:10 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:35:10.764 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:10 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:10.765 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:10 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:35:10.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:10 smithi134 ceph-mon[25063]: pgmap v219: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-05-08T06:35:10.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:10 smithi134 ceph-mon[25063]: mds.? [v2:172.21.15.82:6834/2448674594,v1:172.21.15.82:6835/2448674594] up:active 2024-05-08T06:35:10.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:10 smithi134 ceph-mon[25063]: fsmap foofs:1 {0=foofs.smithi082.ofnfgy=up:active} 2024-05-08T06:35:10.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:10 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:10.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:10 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:35:10.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:10 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:10.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:10 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:35:11.402 INFO:tasks.cephadm:mds.foofs has 0/2 2024-05-08T06:35:11.998 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:11 smithi134 ceph-mon[25063]: mds.? [v2:172.21.15.134:6832/3452736139,v1:172.21.15.134:6833/3452736139] up:boot 2024-05-08T06:35:11.998 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:11 smithi134 ceph-mon[25063]: fsmap foofs:1 {0=foofs.smithi082.ofnfgy=up:active} 1 up:standby 2024-05-08T06:35:11.998 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:11 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi134.ooovgp"}]: dispatch 2024-05-08T06:35:11.998 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:11 smithi134 ceph-mon[25063]: fsmap foofs:1 {0=foofs.smithi082.ofnfgy=up:active} 1 up:standby 2024-05-08T06:35:12.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:11 smithi082 ceph-mon[22957]: mds.? [v2:172.21.15.134:6832/3452736139,v1:172.21.15.134:6833/3452736139] up:boot 2024-05-08T06:35:12.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:11 smithi082 ceph-mon[22957]: fsmap foofs:1 {0=foofs.smithi082.ofnfgy=up:active} 1 up:standby 2024-05-08T06:35:12.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:11 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi134.ooovgp"}]: dispatch 2024-05-08T06:35:12.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:11 smithi082 ceph-mon[22957]: fsmap foofs:1 {0=foofs.smithi082.ofnfgy=up:active} 1 up:standby 2024-05-08T06:35:12.403 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph orch ls -f json 2024-05-08T06:35:13.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:12 smithi134 ceph-mon[25063]: from='client.14492 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-08T06:35:13.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:12 smithi134 ceph-mon[25063]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 3 op/s 2024-05-08T06:35:13.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:12 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:13.059 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:12 smithi082 ceph-mon[22957]: from='client.14492 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-08T06:35:13.059 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:12 smithi082 ceph-mon[22957]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 3 op/s 2024-05-08T06:35:13.059 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:12 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:13.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:13 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:14.030 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:13 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:14.982 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:14 smithi082 ceph-mon[22957]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-05-08T06:35:15.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:14 smithi134 ceph-mon[25063]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-05-08T06:35:15.121 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:35:15.121 INFO:teuthology.orchestra.run.smithi082.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-05-08T06:28:37.520432Z", "last_refresh": "2024-05-08T06:32:21.990821Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-05-08T06:28:31.461121Z", "last_refresh": "2024-05-08T06:32:21.990885Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-05-08T06:28:34.343205Z", "last_refresh": "2024-05-08T06:32:21.990977Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-05-08T06:35:03.733050Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-05-08T06:35:03.728027Z", "last_refresh": "2024-05-08T06:35:12.697913Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-05-08T06:28:29.448705Z", "last_refresh": "2024-05-08T06:32:21.990751Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi082:172.21.15.82=smithi082", "smithi134:172.21.15.134=smithi134"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-05-08T06:29:21.871524Z", "last_refresh": "2024-05-08T06:32:21.990611Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-05-08T06:28:35.536726Z", "last_refresh": "2024-05-08T06:32:21.991041Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-05-08T06:32:21.991166Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-05-08T06:28:32.598691Z", "last_refresh": "2024-05-08T06:32:21.991103Z", "ports": [9095], "running": 1, "size": 1}}] 2024-05-08T06:35:15.632 INFO:tasks.cephadm:mds.foofs has 1/2 2024-05-08T06:35:16.633 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- ceph orch ls -f json 2024-05-08T06:35:16.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:16 smithi082 ceph-mon[22957]: from='client.14496 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-08T06:35:16.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:16 smithi082 ceph-mon[22957]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-05-08T06:35:17.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:16 smithi134 ceph-mon[25063]: from='client.14496 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-08T06:35:17.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:16 smithi134 ceph-mon[25063]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-05-08T06:35:18.835 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:35:18.835 INFO:teuthology.orchestra.run.smithi082.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-05-08T06:28:37.520432Z", "last_refresh": "2024-05-08T06:35:17.824388Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-05-08T06:28:31.461121Z", "last_refresh": "2024-05-08T06:35:12.697313Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-05-08T06:28:34.343205Z", "last_refresh": "2024-05-08T06:35:17.824530Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-05-08T06:35:03.733050Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-05-08T06:35:03.728027Z", "last_refresh": "2024-05-08T06:35:12.697913Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-05-08T06:28:29.448705Z", "last_refresh": "2024-05-08T06:35:12.697455Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi082:172.21.15.82=smithi082", "smithi134:172.21.15.134=smithi134"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-05-08T06:29:21.871524Z", "last_refresh": "2024-05-08T06:35:12.697529Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-05-08T06:28:35.536726Z", "last_refresh": "2024-05-08T06:35:12.697595Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-05-08T06:35:12.697658Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-05-08T06:28:32.598691Z", "last_refresh": "2024-05-08T06:35:17.824649Z", "ports": [9095], "running": 1, "size": 1}}] 2024-05-08T06:35:19.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:18 smithi134 ceph-mon[25063]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-05-08T06:35:19.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:18 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:19.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:18 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:19.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:18 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:19.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:18 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:35:19.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:18 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:19.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:18 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:35:19.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:18 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:19.145 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:18 smithi082 ceph-mon[22957]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-05-08T06:35:19.146 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:18 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:19.146 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:18 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:19.146 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:18 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:19.146 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:18 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:35:19.146 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:18 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:19.146 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:18 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:35:19.146 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:18 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:19.519 INFO:tasks.cephadm:mds.foofs has 2/2 2024-05-08T06:35:19.519 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-08T06:35:19.530 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi082.front.sepia.ceph.com 2024-05-08T06:35:19.531 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-05-08T06:35:21.007 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:20 smithi082 ceph-mon[22957]: from='client.14500 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-08T06:35:21.007 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:20 smithi082 ceph-mon[22957]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-05-08T06:35:21.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:20 smithi134 ceph-mon[25063]: from='client.14500 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-08T06:35:21.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:20 smithi134 ceph-mon[25063]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-05-08T06:35:22.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:21 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-05-08T06:35:22.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:21 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-05-08T06:35:22.735 INFO:teuthology.orchestra.run.smithi082.stdout:NFS Cluster Created Successfully 2024-05-08T06:35:22.960 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:22 smithi082 ceph-mon[22957]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:22.960 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:22 smithi082 ceph-mon[22957]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-05-08T06:35:22.960 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:22 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-05-08T06:35:22.960 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:22 smithi082 ceph-mon[22957]: osdmap e66: 8 total, 8 up, 8 in 2024-05-08T06:35:22.961 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:22 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-05-08T06:35:22.961 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:22 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:23.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:22 smithi134 ceph-mon[25063]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:23.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:22 smithi134 ceph-mon[25063]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-05-08T06:35:23.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:22 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-05-08T06:35:23.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:22 smithi134 ceph-mon[25063]: osdmap e66: 8 total, 8 up, 8 in 2024-05-08T06:35:23.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:22 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-05-08T06:35:23.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:22 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:23.449 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-05-08T06:35:23.956 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-05-08T06:35:23.957 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:23 smithi082 ceph-mon[22957]: osdmap e67: 8 total, 8 up, 8 in 2024-05-08T06:35:23.957 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:23.957 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:35:23.957 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:23.957 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:35:23.957 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:23.957 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:23.957 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi082.dfhwyg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-05-08T06:35:23.957 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi082.dfhwyg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-05-08T06:35:23.957 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-05-08T06:35:23.957 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-05-08T06:35:23.957 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:23.957 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-05-08T06:35:23.957 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-05-08T06:35:23.957 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi082.dfhwyg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-08T06:35:23.958 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi082.dfhwyg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-08T06:35:23.958 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:23 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:24.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-05-08T06:35:24.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:23 smithi134 ceph-mon[25063]: osdmap e67: 8 total, 8 up, 8 in 2024-05-08T06:35:24.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:24.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:35:24.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:24.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:35:24.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:24.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:24.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi082.dfhwyg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-05-08T06:35:24.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi082.dfhwyg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-05-08T06:35:24.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-05-08T06:35:24.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-05-08T06:35:24.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:24.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-05-08T06:35:24.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-05-08T06:35:24.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi082.dfhwyg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-08T06:35:24.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi082.dfhwyg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-08T06:35:24.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:23 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:24.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:24 smithi082 ceph-mon[22957]: Saving service nfs.foo spec with placement count:2 2024-05-08T06:35:24.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:24 smithi082 ceph-mon[22957]: Creating key for client.nfs.foo.0.0.smithi082.dfhwyg 2024-05-08T06:35:24.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:24 smithi082 ceph-mon[22957]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-05-08T06:35:24.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:24 smithi082 ceph-mon[22957]: Rados config object exists: conf-nfs.foo 2024-05-08T06:35:24.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:24 smithi082 ceph-mon[22957]: Creating key for client.nfs.foo.0.0.smithi082.dfhwyg-rgw 2024-05-08T06:35:24.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:24 smithi082 ceph-mon[22957]: Deploying daemon nfs.foo.0.0.smithi082.dfhwyg on smithi082 2024-05-08T06:35:24.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:24 smithi082 ceph-mon[22957]: pgmap v228: 97 pgs: 1 creating+peering, 65 active+clean, 31 unknown; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-05-08T06:35:24.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:24 smithi082 ceph-mon[22957]: osdmap e68: 8 total, 8 up, 8 in 2024-05-08T06:35:24.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:24 smithi082 ceph-mon[22957]: mgrmap e20: smithi082.bsnvtb(active, since 5m), standbys: smithi134.zbsjta 2024-05-08T06:35:25.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:24 smithi134 ceph-mon[25063]: Saving service nfs.foo spec with placement count:2 2024-05-08T06:35:25.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:24 smithi134 ceph-mon[25063]: Creating key for client.nfs.foo.0.0.smithi082.dfhwyg 2024-05-08T06:35:25.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:24 smithi134 ceph-mon[25063]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-05-08T06:35:25.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:24 smithi134 ceph-mon[25063]: Rados config object exists: conf-nfs.foo 2024-05-08T06:35:25.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:24 smithi134 ceph-mon[25063]: Creating key for client.nfs.foo.0.0.smithi082.dfhwyg-rgw 2024-05-08T06:35:25.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:24 smithi134 ceph-mon[25063]: Deploying daemon nfs.foo.0.0.smithi082.dfhwyg on smithi082 2024-05-08T06:35:25.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:24 smithi134 ceph-mon[25063]: pgmap v228: 97 pgs: 1 creating+peering, 65 active+clean, 31 unknown; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-05-08T06:35:25.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:24 smithi134 ceph-mon[25063]: osdmap e68: 8 total, 8 up, 8 in 2024-05-08T06:35:25.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:24 smithi134 ceph-mon[25063]: mgrmap e20: smithi082.bsnvtb(active, since 5m), standbys: smithi134.zbsjta 2024-05-08T06:35:26.346 INFO:teuthology.orchestra.run.smithi082.stderr:Invalid command: Unexpected argument '--clusterid' 2024-05-08T06:35:26.346 INFO:teuthology.orchestra.run.smithi082.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-05-08T06:35:26.346 INFO:teuthology.orchestra.run.smithi082.stderr:Error EINVAL: invalid command 2024-05-08T06:35:26.697 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2024-05-08T06:35:26.697 INFO:teuthology.orchestra.run.smithi082.stdout: "bind": "/fake", 2024-05-08T06:35:26.697 INFO:teuthology.orchestra.run.smithi082.stdout: "fs": "foofs", 2024-05-08T06:35:26.697 INFO:teuthology.orchestra.run.smithi082.stdout: "path": "/", 2024-05-08T06:35:26.697 INFO:teuthology.orchestra.run.smithi082.stdout: "cluster": "foo", 2024-05-08T06:35:26.697 INFO:teuthology.orchestra.run.smithi082.stdout: "mode": "RW" 2024-05-08T06:35:26.698 INFO:teuthology.orchestra.run.smithi082.stdout:} 2024-05-08T06:35:26.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:26 smithi082 ceph-mon[22957]: pgmap v230: 97 pgs: 1 creating+peering, 72 active+clean, 24 unknown; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-05-08T06:35:26.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:26 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-05-08T06:35:26.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:26 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-05-08T06:35:27.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:26 smithi134 ceph-mon[25063]: pgmap v230: 97 pgs: 1 creating+peering, 72 active+clean, 24 unknown; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-05-08T06:35:27.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:26 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-05-08T06:35:27.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:26 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-05-08T06:35:27.529 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-05-08T06:35:27.780 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:27 smithi082 ceph-mon[22957]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:27.780 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:27 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:27.781 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:27 smithi082 ceph-mon[22957]: Creating key for client.nfs.foo.1.0.smithi134.xhqvcd 2024-05-08T06:35:27.781 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:27 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi134.xhqvcd", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-05-08T06:35:27.781 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:27 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi134.xhqvcd", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-05-08T06:35:27.781 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:27 smithi082 ceph-mon[22957]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-05-08T06:35:27.781 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:27 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-05-08T06:35:27.781 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:27 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-05-08T06:35:27.781 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:27 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:27.781 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:27 smithi082 ceph-mon[22957]: pgmap v231: 97 pgs: 1 creating+peering, 96 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-05-08T06:35:28.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:27 smithi134 ceph-mon[25063]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:28.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:27 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:28.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:27 smithi134 ceph-mon[25063]: Creating key for client.nfs.foo.1.0.smithi134.xhqvcd 2024-05-08T06:35:28.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:27 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi134.xhqvcd", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-05-08T06:35:28.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:27 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi134.xhqvcd", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-05-08T06:35:28.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:27 smithi134 ceph-mon[25063]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-05-08T06:35:28.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:27 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-05-08T06:35:28.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:27 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-05-08T06:35:28.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:27 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:28.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:27 smithi134 ceph-mon[25063]: pgmap v231: 97 pgs: 1 creating+peering, 96 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-05-08T06:35:29.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:29 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-05-08T06:35:29.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:29 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-05-08T06:35:29.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:29 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi134.xhqvcd-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-08T06:35:29.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:29 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-05-08T06:35:29.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:29 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-05-08T06:35:29.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:29 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi134.xhqvcd-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-08T06:35:30.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:30 smithi082 ceph-mon[22957]: from='client.14542 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:30.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:30 smithi082 ceph-mon[22957]: Rados config object exists: conf-nfs.foo 2024-05-08T06:35:30.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:30 smithi082 ceph-mon[22957]: Creating key for client.nfs.foo.1.0.smithi134.xhqvcd-rgw 2024-05-08T06:35:30.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi134.xhqvcd-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-08T06:35:30.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:30 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:30.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:30 smithi082 ceph-mon[22957]: Deploying daemon nfs.foo.1.0.smithi134.xhqvcd on smithi134 2024-05-08T06:35:30.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:30 smithi082 ceph-mon[22957]: pgmap v232: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 128 B/s rd, 257 B/s wr, 0 op/s 2024-05-08T06:35:30.922 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:30 smithi134 ceph-mon[25063]: from='client.14542 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:30.922 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:30 smithi134 ceph-mon[25063]: Rados config object exists: conf-nfs.foo 2024-05-08T06:35:30.922 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:30 smithi134 ceph-mon[25063]: Creating key for client.nfs.foo.1.0.smithi134.xhqvcd-rgw 2024-05-08T06:35:30.922 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi134.xhqvcd-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-08T06:35:30.922 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:30 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:30.922 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:30 smithi134 ceph-mon[25063]: Deploying daemon nfs.foo.1.0.smithi134.xhqvcd on smithi134 2024-05-08T06:35:30.922 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:30 smithi134 ceph-mon[25063]: pgmap v232: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 128 B/s rd, 257 B/s wr, 0 op/s 2024-05-08T06:35:32.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:32 smithi134 ceph-mon[25063]: from='client.14556 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:32.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:32 smithi134 ceph-mon[25063]: pgmap v233: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.7 KiB/s wr, 3 op/s 2024-05-08T06:35:32.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:32 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:32.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:32 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:35:32.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:32 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:32.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:32 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:35:32.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:32 smithi082 ceph-mon[22957]: from='client.14556 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:32.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:32 smithi082 ceph-mon[22957]: pgmap v233: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.7 KiB/s wr, 3 op/s 2024-05-08T06:35:32.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:32 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:32.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:32 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:35:32.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:32 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:32.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:32 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:35:33.989 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:33 smithi082 ceph-mon[22957]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:33.989 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:33 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:34.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:33 smithi134 ceph-mon[25063]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:34.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:33 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:34.868 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:34 smithi082 ceph-mon[22957]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:34.868 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:34 smithi082 ceph-mon[22957]: pgmap v234: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-05-08T06:35:35.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:34 smithi134 ceph-mon[25063]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:35.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:34 smithi134 ceph-mon[25063]: pgmap v234: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-05-08T06:35:36.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:35 smithi134 ceph-mon[25063]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:36.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:35 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:36.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:35 smithi082 ceph-mon[22957]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:36.167 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:35 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:36.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:36 smithi082 ceph-mon[22957]: pgmap v235: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.9 KiB/s wr, 5 op/s 2024-05-08T06:35:36.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:36 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:37.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:36 smithi134 ceph-mon[25063]: pgmap v235: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.9 KiB/s wr, 5 op/s 2024-05-08T06:35:37.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:36 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:37.309 INFO:teuthology.orchestra.run.smithi082.stdout:nfs.foo ?:2049 2/2 2s ago 14s count:2 2024-05-08T06:35:37.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:37 smithi082 ceph-mon[22957]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:37.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:37.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:37.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:35:37.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:37.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:35:37.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:37.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:37.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2024-05-08T06:35:37.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-05-08T06:35:37.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-05-08T06:35:37.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-05-08T06:35:37.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:35:37.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:35:37.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:38.000 INFO:teuthology.run_tasks:Running task vip.exec... 2024-05-08T06:35:38.010 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi082.front.sepia.ceph.com 2024-05-08T06:35:38.010 DEBUG:teuthology.orchestra.run.smithi082:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-05-08T06:35:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:37 smithi134 ceph-mon[25063]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:35:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:35:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:35:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2024-05-08T06:35:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-05-08T06:35:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-05-08T06:35:38.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-05-08T06:35:38.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:35:38.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:35:38.029 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:35:38.035 INFO:teuthology.orchestra.run.smithi082.stderr:+ mkdir /mnt/foo 2024-05-08T06:35:38.058 DEBUG:teuthology.orchestra.run.smithi082:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-05-08T06:35:38.125 INFO:teuthology.orchestra.run.smithi082.stderr:++ hostname 2024-05-08T06:35:38.125 INFO:teuthology.orchestra.run.smithi082.stderr:+ mount -t nfs smithi082:/fake /mnt/foo -o sync 2024-05-08T06:35:38.834 DEBUG:teuthology.orchestra.run.smithi082:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-05-08T06:35:38.899 INFO:teuthology.orchestra.run.smithi082.stderr:+ echo test 2024-05-08T06:35:38.910 DEBUG:teuthology.orchestra.run.smithi082:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-05-08T06:35:38.971 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:38 smithi082 ceph-mon[22957]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:38.973 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:38 smithi082 ceph-mon[22957]: pgmap v236: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-05-08T06:35:38.973 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]': finished 2024-05-08T06:35:38.973 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-05-08T06:35:38.973 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-05-08T06:35:38.973 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:38 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-05-08T06:35:38.973 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:38 smithi082 ceph-mon[22957]: osdmap e69: 8 total, 8 up, 8 in 2024-05-08T06:35:38.974 INFO:teuthology.orchestra.run.smithi082.stderr:+ sync 2024-05-08T06:35:39.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:38 smithi134 ceph-mon[25063]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:35:39.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:38 smithi134 ceph-mon[25063]: pgmap v236: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-05-08T06:35:39.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:38 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]': finished 2024-05-08T06:35:39.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:38 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-05-08T06:35:39.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:38 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-05-08T06:35:39.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:38 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-05-08T06:35:39.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:38 smithi134 ceph-mon[25063]: osdmap e69: 8 total, 8 up, 8 in 2024-05-08T06:35:39.181 INFO:teuthology.run_tasks:Running task parallel... 2024-05-08T06:35:39.191 INFO:teuthology.task.parallel:starting parallel... 2024-05-08T06:35:39.191 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-05-08T06:35:39.192 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-05-08T06:35:39.192 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi082.front.sepia.ceph.com 2024-05-08T06:35:39.192 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -e sha1=7c8f650b36e258f639fa4a83becade57cbfd2009 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-05-08T06:35:39.193 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-05-08T06:35:39.193 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-05-08T06:35:39.193 INFO:teuthology.task.exec:Executing custom commands... 2024-05-08T06:35:39.194 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi082.front.sepia.ceph.com 2024-05-08T06:35:39.194 DEBUG:teuthology.orchestra.run.smithi082:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-05-08T06:35:39.315 INFO:teuthology.orchestra.run.smithi082.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-05-08T06:35:39.315 INFO:teuthology.orchestra.run.smithi082.stdout: 2024-05-08T06:35:39.315 INFO:teuthology.orchestra.run.smithi082.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-05-08T06:35:39.316 INFO:teuthology.orchestra.run.smithi082.stdout:3 of 5 processes prepared for launch 0 sec 2024-05-08T06:35:39.416 INFO:teuthology.orchestra.run.smithi082.stdout:5 of 5 processes prepared for launch 0 sec 2024-05-08T06:35:39.417 INFO:teuthology.orchestra.run.smithi082.stdout:releasing clients 2024-05-08T06:35:39.784 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:39 smithi082 ceph-mon[22957]: osdmap e70: 8 total, 8 up, 8 in 2024-05-08T06:35:39.784 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:39 smithi082 ceph-mon[22957]: pgmap v239: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-05-08T06:35:40.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:39 smithi134 ceph-mon[25063]: osdmap e70: 8 total, 8 up, 8 in 2024-05-08T06:35:40.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:39 smithi134 ceph-mon[25063]: pgmap v239: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-05-08T06:35:40.417 INFO:teuthology.orchestra.run.smithi082.stdout: 5 134 29.40 MB/sec warmup 1 sec latency 53.918 ms 2024-05-08T06:35:41.417 INFO:teuthology.orchestra.run.smithi082.stdout: 5 263 24.78 MB/sec warmup 2 sec latency 31.701 ms 2024-05-08T06:35:42.417 INFO:teuthology.orchestra.run.smithi082.stdout: 5 406 23.93 MB/sec warmup 3 sec latency 36.905 ms 2024-05-08T06:35:42.601 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -e sha1=7c8f650b36e258f639fa4a83becade57cbfd2009 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-05-08T06:35:42.957 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:42 smithi082 ceph-mon[22957]: pgmap v240: 97 pgs: 97 active+clean; 30 MiB data, 65 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.7 MiB/s wr, 155 op/s; 0 B/s, 0 objects/s recovering 2024-05-08T06:35:43.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:42 smithi134 ceph-mon[25063]: pgmap v240: 97 pgs: 97 active+clean; 30 MiB data, 65 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.7 MiB/s wr, 155 op/s; 0 B/s, 0 objects/s recovering 2024-05-08T06:35:43.417 INFO:teuthology.orchestra.run.smithi082.stdout: 5 538 23.39 MB/sec warmup 4 sec latency 32.548 ms 2024-05-08T06:35:44.417 INFO:teuthology.orchestra.run.smithi082.stdout: 5 877 21.65 MB/sec warmup 5 sec latency 31.361 ms 2024-05-08T06:35:44.728 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:44 smithi082 ceph-mon[22957]: pgmap v241: 97 pgs: 97 active+clean; 30 MiB data, 65 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 3.7 MiB/s wr, 151 op/s; 0 B/s, 0 objects/s recovering 2024-05-08T06:35:45.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:44 smithi134 ceph-mon[25063]: pgmap v241: 97 pgs: 97 active+clean; 30 MiB data, 65 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 3.7 MiB/s wr, 151 op/s; 0 B/s, 0 objects/s recovering 2024-05-08T06:35:45.417 INFO:teuthology.orchestra.run.smithi082.stdout: 5 1359 20.71 MB/sec warmup 6 sec latency 31.600 ms 2024-05-08T06:35:46.417 INFO:teuthology.orchestra.run.smithi082.stdout: 5 2360 19.94 MB/sec warmup 7 sec latency 30.438 ms 2024-05-08T06:35:46.737 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:46 smithi082 ceph-mon[22957]: pgmap v242: 97 pgs: 97 active+clean; 57 MiB data, 104 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 7.4 MiB/s wr, 365 op/s; 0 B/s, 0 objects/s recovering 2024-05-08T06:35:47.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:46 smithi134 ceph-mon[25063]: pgmap v242: 97 pgs: 97 active+clean; 57 MiB data, 104 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 7.4 MiB/s wr, 365 op/s; 0 B/s, 0 objects/s recovering 2024-05-08T06:35:47.304 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -e sha1=7c8f650b36e258f639fa4a83becade57cbfd2009 -- bash -c 'ceph config set global log_to_journald false --force' 2024-05-08T06:35:47.417 INFO:teuthology.orchestra.run.smithi082.stdout: 5 3289 22.01 MB/sec warmup 8 sec latency 19.382 ms 2024-05-08T06:35:47.742 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:47 smithi082 ceph-mon[22957]: pgmap v243: 97 pgs: 97 active+clean; 111 MiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 12 MiB/s wr, 547 op/s; 0 B/s, 0 objects/s recovering 2024-05-08T06:35:48.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:47 smithi134 ceph-mon[25063]: pgmap v243: 97 pgs: 97 active+clean; 111 MiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 12 MiB/s wr, 547 op/s; 0 B/s, 0 objects/s recovering 2024-05-08T06:35:48.417 INFO:teuthology.orchestra.run.smithi082.stdout: 5 3858 22.12 MB/sec warmup 9 sec latency 22.193 ms 2024-05-08T06:35:49.417 INFO:teuthology.orchestra.run.smithi082.stdout: 5 4481 20.51 MB/sec warmup 10 sec latency 22.476 ms 2024-05-08T06:35:50.018 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:49 smithi082 ceph-mon[22957]: pgmap v244: 97 pgs: 97 active+clean; 111 MiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 11 MiB/s wr, 497 op/s; 0 B/s, 0 objects/s recovering 2024-05-08T06:35:50.027 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:49 smithi134 ceph-mon[25063]: pgmap v244: 97 pgs: 97 active+clean; 111 MiB data, 186 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 11 MiB/s wr, 497 op/s; 0 B/s, 0 objects/s recovering 2024-05-08T06:35:50.417 INFO:teuthology.orchestra.run.smithi082.stdout: 5 5180 20.58 MB/sec warmup 11 sec latency 146.252 ms 2024-05-08T06:35:51.417 INFO:teuthology.orchestra.run.smithi082.stdout: 5 5929 19.58 MB/sec warmup 12 sec latency 37.069 ms 2024-05-08T06:35:52.174 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -e sha1=7c8f650b36e258f639fa4a83becade57cbfd2009 -- bash -c 'ceph mgr module enable nfs --force' 2024-05-08T06:35:52.417 INFO:teuthology.orchestra.run.smithi082.stdout: 5 6692 20.48 MB/sec warmup 13 sec latency 49.016 ms 2024-05-08T06:35:52.791 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:52 smithi082 ceph-mon[22957]: pgmap v245: 97 pgs: 97 active+clean; 131 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 14 MiB/s wr, 861 op/s; 0 B/s, 0 objects/s recovering 2024-05-08T06:35:53.027 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:52 smithi134 ceph-mon[25063]: pgmap v245: 97 pgs: 97 active+clean; 131 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 14 MiB/s wr, 861 op/s; 0 B/s, 0 objects/s recovering 2024-05-08T06:35:53.417 INFO:teuthology.orchestra.run.smithi082.stdout: 5 7383 21.04 MB/sec warmup 14 sec latency 21.235 ms 2024-05-08T06:35:54.417 INFO:teuthology.orchestra.run.smithi082.stdout: 5 7969 19.99 MB/sec warmup 15 sec latency 21.844 ms 2024-05-08T06:35:54.824 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:54 smithi082 ceph-mon[22957]: pgmap v246: 97 pgs: 97 active+clean; 131 MiB data, 262 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 12 MiB/s wr, 762 op/s 2024-05-08T06:35:55.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:54 smithi134 ceph-mon[25063]: pgmap v246: 97 pgs: 97 active+clean; 131 MiB data, 262 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 12 MiB/s wr, 762 op/s 2024-05-08T06:35:55.417 INFO:teuthology.orchestra.run.smithi082.stdout: 5 8690 19.95 MB/sec warmup 16 sec latency 21.731 ms 2024-05-08T06:35:56.417 INFO:teuthology.orchestra.run.smithi082.stdout: 5 9454 19.39 MB/sec warmup 17 sec latency 33.521 ms 2024-05-08T06:35:56.737 INFO:teuthology.orchestra.run.smithi082.stderr:module 'nfs' is already enabled 2024-05-08T06:35:56.791 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:56 smithi082 ceph-mon[22957]: pgmap v247: 97 pgs: 97 active+clean; 134 MiB data, 282 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 13 MiB/s wr, 889 op/s 2024-05-08T06:35:56.791 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:56 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/2636484481' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-05-08T06:35:57.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:56 smithi134 ceph-mon[25063]: pgmap v247: 97 pgs: 97 active+clean; 134 MiB data, 282 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 13 MiB/s wr, 889 op/s 2024-05-08T06:35:57.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:56 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/2636484481' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-05-08T06:35:57.417 INFO:teuthology.orchestra.run.smithi082.stdout: 5 10091 19.89 MB/sec warmup 18 sec latency 23.676 ms 2024-05-08T06:35:57.795 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:57 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/2636484481' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-05-08T06:35:57.795 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:57 smithi082 ceph-mon[22957]: mgrmap e21: smithi082.bsnvtb(active, since 6m), standbys: smithi134.zbsjta 2024-05-08T06:35:57.795 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:57 smithi082 ceph-mon[22957]: pgmap v248: 97 pgs: 97 active+clean; 145 MiB data, 393 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 14 MiB/s wr, 1.02k op/s 2024-05-08T06:35:57.847 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -e sha1=7c8f650b36e258f639fa4a83becade57cbfd2009 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-05-08T06:35:58.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:57 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/2636484481' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-05-08T06:35:58.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:57 smithi134 ceph-mon[25063]: mgrmap e21: smithi082.bsnvtb(active, since 6m), standbys: smithi134.zbsjta 2024-05-08T06:35:58.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:57 smithi134 ceph-mon[25063]: pgmap v248: 97 pgs: 97 active+clean; 145 MiB data, 393 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 14 MiB/s wr, 1.02k op/s 2024-05-08T06:35:58.425 INFO:teuthology.orchestra.run.smithi082.stdout: 5 10714 20.11 MB/sec warmup 19 sec latency 36.403 ms 2024-05-08T06:35:59.425 INFO:teuthology.orchestra.run.smithi082.stdout: 5 11268 19.62 MB/sec warmup 20 sec latency 22.883 ms 2024-05-08T06:35:59.817 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:35:59 smithi082 ceph-mon[22957]: pgmap v249: 97 pgs: 97 active+clean; 145 MiB data, 393 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 813 op/s 2024-05-08T06:36:00.056 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:35:59 smithi134 ceph-mon[25063]: pgmap v249: 97 pgs: 97 active+clean; 145 MiB data, 393 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 813 op/s 2024-05-08T06:36:00.425 INFO:teuthology.orchestra.run.smithi082.stdout: 5 11831 19.21 MB/sec warmup 21 sec latency 25.427 ms 2024-05-08T06:36:01.336 INFO:teuthology.orchestra.run.smithi082.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:7c8f650b36e258f639fa4a83becade57cbfd2009 2024-05-08T06:36:01.425 INFO:teuthology.orchestra.run.smithi082.stdout: 5 12635 19.11 MB/sec warmup 22 sec latency 21.858 ms 2024-05-08T06:36:02.385 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:02 smithi082 ceph-mon[22957]: from='client.14608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:7c8f650b36e258f639fa4a83becade57cbfd2009", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:36:02.385 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:02 smithi082 ceph-mon[22957]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:7c8f650b36e258f639fa4a83becade57cbfd2009 2024-05-08T06:36:02.385 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:02 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:36:02.385 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:02 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:36:02.385 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:02 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:36:02.385 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:02 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:36:02.385 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:02 smithi082 ceph-mon[22957]: pgmap v250: 97 pgs: 97 active+clean; 155 MiB data, 436 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 13 MiB/s wr, 1.17k op/s 2024-05-08T06:36:02.425 INFO:teuthology.orchestra.run.smithi082.stdout: 5 13211 18.84 MB/sec warmup 23 sec latency 43.067 ms 2024-05-08T06:36:02.600 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-05-08T06:36:02.601 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi082.front.sepia.ceph.com 2024-05-08T06:36:02.601 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -e sha1=7c8f650b36e258f639fa4a83becade57cbfd2009 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-05-08T06:36:02.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:02 smithi134 ceph-mon[25063]: from='client.14608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:7c8f650b36e258f639fa4a83becade57cbfd2009", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:36:02.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:02 smithi134 ceph-mon[25063]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:7c8f650b36e258f639fa4a83becade57cbfd2009 2024-05-08T06:36:02.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:02 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:36:02.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:02 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:36:02.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:02 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:36:02.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:02 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:36:02.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:02 smithi134 ceph-mon[25063]: pgmap v250: 97 pgs: 97 active+clean; 155 MiB data, 436 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 13 MiB/s wr, 1.17k op/s 2024-05-08T06:36:03.425 INFO:teuthology.orchestra.run.smithi082.stdout: 5 13832 19.22 MB/sec warmup 24 sec latency 30.636 ms 2024-05-08T06:36:03.913 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:03 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:36:03.913 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:03 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:36:03.913 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:03 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:36:03.913 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:03 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:36:03.913 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:03 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:36:03.913 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:03 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:36:03.913 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:03 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:36:03.914 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:03 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:36:04.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:03 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:36:04.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:03 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:36:04.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:03 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:36:04.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:03 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:36:04.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:03 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:36:04.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:03 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:36:04.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:03 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:36:04.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:03 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:36:04.426 INFO:teuthology.orchestra.run.smithi082.stdout: 5 14415 19.32 MB/sec warmup 25 sec latency 36.419 ms 2024-05-08T06:36:04.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:04 smithi082 ceph-mon[22957]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:7c8f650b36e258f639fa4a83becade57cbfd2009 2024-05-08T06:36:04.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:04 smithi082 ceph-mon[22957]: Upgrade: Paused due to UPGRADE_FAILED_PULL: Upgrade: failed to pull target image 2024-05-08T06:36:04.917 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:04 smithi082 ceph-mon[22957]: pgmap v251: 97 pgs: 97 active+clean; 155 MiB data, 437 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 8.5 MiB/s wr, 785 op/s 2024-05-08T06:36:04.918 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:04 smithi082 ceph-mon[22957]: Health check failed: Upgrade: failed to pull target image (UPGRADE_FAILED_PULL) 2024-05-08T06:36:05.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:04 smithi134 ceph-mon[25063]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:7c8f650b36e258f639fa4a83becade57cbfd2009 2024-05-08T06:36:05.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:04 smithi134 ceph-mon[25063]: Upgrade: Paused due to UPGRADE_FAILED_PULL: Upgrade: failed to pull target image 2024-05-08T06:36:05.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:04 smithi134 ceph-mon[25063]: pgmap v251: 97 pgs: 97 active+clean; 155 MiB data, 437 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 8.5 MiB/s wr, 785 op/s 2024-05-08T06:36:05.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:04 smithi134 ceph-mon[25063]: Health check failed: Upgrade: failed to pull target image (UPGRADE_FAILED_PULL) 2024-05-08T06:36:05.425 INFO:teuthology.orchestra.run.smithi082.stdout: 5 14874 18.78 MB/sec warmup 26 sec latency 42.900 ms 2024-05-08T06:36:05.538 INFO:teuthology.orchestra.run.smithi082.stdout:true 2024-05-08T06:36:06.041 INFO:teuthology.orchestra.run.smithi082.stdout:"Error: UPGRADE_FAILED_PULL: Upgrade: failed to pull target image" 2024-05-08T06:36:06.426 INFO:teuthology.orchestra.run.smithi082.stdout: 5 15702 18.88 MB/sec warmup 27 sec latency 41.294 ms 2024-05-08T06:36:06.942 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:06 smithi082 ceph-mon[22957]: pgmap v252: 97 pgs: 97 active+clean; 157 MiB data, 431 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 805 op/s 2024-05-08T06:36:06.942 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:06 smithi082 ceph-mon[22957]: from='client.24363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:36:07.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:06 smithi134 ceph-mon[25063]: pgmap v252: 97 pgs: 97 active+clean; 157 MiB data, 431 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 805 op/s 2024-05-08T06:36:07.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:06 smithi134 ceph-mon[25063]: from='client.24363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:36:07.426 INFO:teuthology.orchestra.run.smithi082.stdout: 5 16616 18.77 MB/sec warmup 28 sec latency 24.915 ms 2024-05-08T06:36:07.534 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -e sha1=7c8f650b36e258f639fa4a83becade57cbfd2009 -- bash -c 'ceph orch ps' 2024-05-08T06:36:07.950 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:07 smithi082 ceph-mon[22957]: from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:36:07.950 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:07 smithi082 ceph-mon[22957]: pgmap v253: 97 pgs: 97 active+clean; 169 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 998 op/s 2024-05-08T06:36:07.950 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:07 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:36:08.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:07 smithi134 ceph-mon[25063]: from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:36:08.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:07 smithi134 ceph-mon[25063]: pgmap v253: 97 pgs: 97 active+clean; 169 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 998 op/s 2024-05-08T06:36:08.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:07 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:36:08.426 INFO:teuthology.orchestra.run.smithi082.stdout: 5 17224 19.08 MB/sec warmup 29 sec latency 26.112 ms 2024-05-08T06:36:09.426 INFO:teuthology.orchestra.run.smithi082.stdout: 5 17893 19.23 MB/sec warmup 30 sec latency 28.253 ms 2024-05-08T06:36:10.426 INFO:teuthology.orchestra.run.smithi082.stdout: 5 18374 18.82 MB/sec warmup 31 sec latency 22.145 ms 2024-05-08T06:36:10.721 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:10 smithi082 ceph-mon[22957]: pgmap v254: 97 pgs: 97 active+clean; 169 MiB data, 512 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 717 op/s 2024-05-08T06:36:10.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:10 smithi134 ceph-mon[25063]: pgmap v254: 97 pgs: 97 active+clean; 169 MiB data, 512 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 717 op/s 2024-05-08T06:36:10.988 INFO:teuthology.orchestra.run.smithi082.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-08T06:36:10.988 INFO:teuthology.orchestra.run.smithi082.stdout:alertmanager.smithi082 smithi082 *:9093,9094 running (5m) 34s ago 6m 24.4M - ba2b418f427c 86af446dfdba 2024-05-08T06:36:10.988 INFO:teuthology.orchestra.run.smithi082.stdout:crash.smithi082 smithi082 running (6m) 34s ago 6m 6409k - 17.2.0 e1d6a67b021e c5efbe70843e 2024-05-08T06:36:10.988 INFO:teuthology.orchestra.run.smithi082.stdout:crash.smithi134 smithi134 running (6m) 36s ago 6m 6405k - 17.2.0 e1d6a67b021e c670ee9715d5 2024-05-08T06:36:10.988 INFO:teuthology.orchestra.run.smithi082.stdout:grafana.smithi082 smithi082 *:3000 running (5m) 34s ago 6m 56.3M - 8.3.5 dad864ee21e9 903d0b5aab5f 2024-05-08T06:36:10.989 INFO:teuthology.orchestra.run.smithi082.stdout:mds.foofs.smithi082.ofnfgy smithi082 running (64s) 34s ago 63s 13.3M - 17.2.0 e1d6a67b021e 3ccf22c4807b 2024-05-08T06:36:10.989 INFO:teuthology.orchestra.run.smithi082.stdout:mds.foofs.smithi134.ooovgp smithi134 running (60s) 36s ago 60s 12.1M - 17.2.0 e1d6a67b021e a436346080b2 2024-05-08T06:36:10.989 INFO:teuthology.orchestra.run.smithi082.stdout:mgr.smithi082.bsnvtb smithi082 *:9283 running (8m) 34s ago 8m 460M - 17.2.0 e1d6a67b021e affb5f53b21e 2024-05-08T06:36:10.989 INFO:teuthology.orchestra.run.smithi082.stdout:mgr.smithi134.zbsjta smithi134 *:8443,9283 running (6m) 36s ago 6m 413M - 17.2.0 e1d6a67b021e 971c6830c247 2024-05-08T06:36:10.989 INFO:teuthology.orchestra.run.smithi082.stdout:mon.smithi082 smithi082 running (8m) 34s ago 8m 46.5M 2048M 17.2.0 e1d6a67b021e ee82d4cf1e0f 2024-05-08T06:36:10.989 INFO:teuthology.orchestra.run.smithi082.stdout:mon.smithi134 smithi134 running (6m) 36s ago 6m 37.0M 2048M 17.2.0 e1d6a67b021e 302cfd0d2bb4 2024-05-08T06:36:10.990 INFO:teuthology.orchestra.run.smithi082.stdout:nfs.foo.0.0.smithi082.dfhwyg smithi082 *:2049 running (44s) 34s ago 44s 53.1M - 4.0 e1d6a67b021e 596537e6bfac 2024-05-08T06:36:10.990 INFO:teuthology.orchestra.run.smithi082.stdout:nfs.foo.1.0.smithi134.xhqvcd smithi134 *:2049 running (39s) 36s ago 39s 50.8M - 4.0 e1d6a67b021e 13ef5875be00 2024-05-08T06:36:10.990 INFO:teuthology.orchestra.run.smithi082.stdout:node-exporter.smithi082 smithi082 *:9100 running (6m) 34s ago 6m 16.5M - 1dbe0e931976 3ba57430c975 2024-05-08T06:36:10.990 INFO:teuthology.orchestra.run.smithi082.stdout:node-exporter.smithi134 smithi134 *:9100 running (6m) 36s ago 6m 17.0M - 1dbe0e931976 aea98e24f85a 2024-05-08T06:36:10.990 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0 smithi082 running (5m) 34s ago 5m 45.5M 3216M 17.2.0 e1d6a67b021e 4061f89d770d 2024-05-08T06:36:10.990 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1 smithi082 running (4m) 34s ago 4m 45.7M 3216M 17.2.0 e1d6a67b021e 2c86afbd7dc1 2024-05-08T06:36:10.991 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2 smithi082 running (4m) 34s ago 4m 40.0M 3216M 17.2.0 e1d6a67b021e a703fa1e14f7 2024-05-08T06:36:10.991 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3 smithi082 running (4m) 34s ago 4m 40.8M 3216M 17.2.0 e1d6a67b021e 7f3b8f015f59 2024-05-08T06:36:10.991 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4 smithi134 running (3m) 36s ago 3m 41.3M 3984M 17.2.0 e1d6a67b021e 3de640027e4e 2024-05-08T06:36:10.991 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5 smithi134 running (3m) 36s ago 3m 43.9M 3984M 17.2.0 e1d6a67b021e 75f363c21b07 2024-05-08T06:36:10.991 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6 smithi134 running (2m) 36s ago 2m 38.0M 3984M 17.2.0 e1d6a67b021e 98bcf9455cd6 2024-05-08T06:36:10.991 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7 smithi134 running (2m) 36s ago 2m 41.9M 3984M 17.2.0 e1d6a67b021e 9b1f288b1e8c 2024-05-08T06:36:10.992 INFO:teuthology.orchestra.run.smithi082.stdout:prometheus.smithi082 smithi082 *:9095 running (5m) 34s ago 5m 63.0M - 514e6a882f6e 637ca038131b 2024-05-08T06:36:11.426 INFO:teuthology.orchestra.run.smithi082.stdout: 5 18973 18.62 MB/sec warmup 32 sec latency 21.653 ms 2024-05-08T06:36:12.426 INFO:teuthology.orchestra.run.smithi082.stdout: 5 19792 18.54 MB/sec warmup 33 sec latency 28.160 ms 2024-05-08T06:36:12.733 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:12 smithi082 ceph-mon[22957]: from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:36:12.733 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:12 smithi082 ceph-mon[22957]: pgmap v255: 97 pgs: 97 active+clean; 180 MiB data, 599 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 1.11k op/s 2024-05-08T06:36:13.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:12 smithi134 ceph-mon[25063]: from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:36:13.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:12 smithi134 ceph-mon[25063]: pgmap v255: 97 pgs: 97 active+clean; 180 MiB data, 599 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 1.11k op/s 2024-05-08T06:36:13.104 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -e sha1=7c8f650b36e258f639fa4a83becade57cbfd2009 -- bash -c 'ceph versions' 2024-05-08T06:36:13.426 INFO:teuthology.orchestra.run.smithi082.stdout: 5 20409 18.42 MB/sec warmup 34 sec latency 24.561 ms 2024-05-08T06:36:14.426 INFO:teuthology.orchestra.run.smithi082.stdout: 5 21292 19.01 MB/sec warmup 35 sec latency 32.281 ms 2024-05-08T06:36:14.925 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:14 smithi082 ceph-mon[22957]: pgmap v256: 97 pgs: 97 active+clean; 180 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 8.1 MiB/s wr, 740 op/s 2024-05-08T06:36:15.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:14 smithi134 ceph-mon[25063]: pgmap v256: 97 pgs: 97 active+clean; 180 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 8.1 MiB/s wr, 740 op/s 2024-05-08T06:36:15.426 INFO:teuthology.orchestra.run.smithi082.stdout: 5 21848 18.87 MB/sec warmup 36 sec latency 22.345 ms 2024-05-08T06:36:16.426 INFO:teuthology.orchestra.run.smithi082.stdout: 5 22457 18.62 MB/sec warmup 37 sec latency 30.638 ms 2024-05-08T06:36:16.496 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2024-05-08T06:36:16.496 INFO:teuthology.orchestra.run.smithi082.stdout: "mon": { 2024-05-08T06:36:16.496 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-08T06:36:16.496 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2024-05-08T06:36:16.496 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr": { 2024-05-08T06:36:16.496 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-08T06:36:16.496 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2024-05-08T06:36:16.496 INFO:teuthology.orchestra.run.smithi082.stdout: "osd": { 2024-05-08T06:36:16.496 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-08T06:36:16.496 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2024-05-08T06:36:16.496 INFO:teuthology.orchestra.run.smithi082.stdout: "mds": { 2024-05-08T06:36:16.496 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-08T06:36:16.497 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2024-05-08T06:36:16.497 INFO:teuthology.orchestra.run.smithi082.stdout: "overall": { 2024-05-08T06:36:16.497 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-05-08T06:36:16.497 INFO:teuthology.orchestra.run.smithi082.stdout: } 2024-05-08T06:36:16.497 INFO:teuthology.orchestra.run.smithi082.stdout:} 2024-05-08T06:36:16.945 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:16 smithi082 ceph-mon[22957]: pgmap v257: 97 pgs: 97 active+clean; 179 MiB data, 613 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 742 op/s 2024-05-08T06:36:16.945 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:16 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3672631342' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-08T06:36:17.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:16 smithi134 ceph-mon[25063]: pgmap v257: 97 pgs: 97 active+clean; 179 MiB data, 613 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 742 op/s 2024-05-08T06:36:17.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:16 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/3672631342' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-08T06:36:17.426 INFO:teuthology.orchestra.run.smithi082.stdout: 5 23176 18.60 MB/sec warmup 38 sec latency 25.528 ms 2024-05-08T06:36:17.884 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -e sha1=7c8f650b36e258f639fa4a83becade57cbfd2009 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-05-08T06:36:18.426 INFO:teuthology.orchestra.run.smithi082.stdout: 5 23859 18.46 MB/sec warmup 39 sec latency 29.180 ms 2024-05-08T06:36:18.961 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:18 smithi082 ceph-mon[22957]: pgmap v258: 97 pgs: 97 active+clean; 188 MiB data, 682 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2024-05-08T06:36:19.027 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:18 smithi134 ceph-mon[25063]: pgmap v258: 97 pgs: 97 active+clean; 188 MiB data, 682 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2024-05-08T06:36:19.426 INFO:teuthology.orchestra.run.smithi082.stdout: 5 24563 18.75 MB/sec warmup 40 sec latency 24.746 ms 2024-05-08T06:36:20.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 25101 18.78 MB/sec warmup 41 sec latency 48.637 ms 2024-05-08T06:36:20.604 INFO:teuthology.orchestra.run.smithi082.stdout:wait for servicemap items w/ changing names to refresh 2024-05-08T06:36:20.750 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:20 smithi082 ceph-mon[22957]: pgmap v259: 97 pgs: 97 active+clean; 192 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 8.5 MiB/s wr, 807 op/s 2024-05-08T06:36:21.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:20 smithi134 ceph-mon[25063]: pgmap v259: 97 pgs: 97 active+clean; 192 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 8.5 MiB/s wr, 807 op/s 2024-05-08T06:36:21.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 25439 18.45 MB/sec warmup 42 sec latency 45.606 ms 2024-05-08T06:36:21.558 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -e sha1=7c8f650b36e258f639fa4a83becade57cbfd2009 -- bash -c 'sleep 60' 2024-05-08T06:36:22.426 INFO:teuthology.orchestra.run.smithi082.stdout: 5 25936 18.21 MB/sec warmup 43 sec latency 43.328 ms 2024-05-08T06:36:22.793 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:22 smithi082 ceph-mon[22957]: pgmap v260: 97 pgs: 97 active+clean; 198 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 1.12k op/s 2024-05-08T06:36:23.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:22 smithi134 ceph-mon[25063]: pgmap v260: 97 pgs: 97 active+clean; 198 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 1.12k op/s 2024-05-08T06:36:23.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 26672 18.23 MB/sec warmup 44 sec latency 50.523 ms 2024-05-08T06:36:24.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 27418 18.11 MB/sec warmup 45 sec latency 22.167 ms 2024-05-08T06:36:24.805 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:24 smithi082 ceph-mon[22957]: pgmap v261: 97 pgs: 97 active+clean; 198 MiB data, 784 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.9 MiB/s wr, 737 op/s 2024-05-08T06:36:25.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:24 smithi134 ceph-mon[25063]: pgmap v261: 97 pgs: 97 active+clean; 198 MiB data, 784 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.9 MiB/s wr, 737 op/s 2024-05-08T06:36:25.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 28107 18.37 MB/sec warmup 46 sec latency 23.196 ms 2024-05-08T06:36:26.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 28735 18.48 MB/sec warmup 47 sec latency 24.977 ms 2024-05-08T06:36:26.842 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:26 smithi082 ceph-mon[22957]: pgmap v262: 97 pgs: 97 active+clean; 198 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 8.2 MiB/s wr, 755 op/s 2024-05-08T06:36:27.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:26 smithi134 ceph-mon[25063]: pgmap v262: 97 pgs: 97 active+clean; 198 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 8.2 MiB/s wr, 755 op/s 2024-05-08T06:36:27.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 29316 18.20 MB/sec warmup 48 sec latency 30.632 ms 2024-05-08T06:36:28.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 29978 18.18 MB/sec warmup 49 sec latency 33.729 ms 2024-05-08T06:36:28.859 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:28 smithi082 ceph-mon[22957]: pgmap v263: 97 pgs: 97 active+clean; 208 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 9.1 KiB/s rd, 11 MiB/s wr, 1.07k op/s 2024-05-08T06:36:29.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:28 smithi134 ceph-mon[25063]: pgmap v263: 97 pgs: 97 active+clean; 208 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 9.1 KiB/s rd, 11 MiB/s wr, 1.07k op/s 2024-05-08T06:36:29.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 30781 18.14 MB/sec warmup 50 sec latency 21.540 ms 2024-05-08T06:36:29.873 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:29 smithi082 ceph-mon[22957]: pgmap v264: 97 pgs: 97 active+clean; 208 MiB data, 867 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.8 MiB/s wr, 750 op/s 2024-05-08T06:36:30.278 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:29 smithi134 ceph-mon[25063]: pgmap v264: 97 pgs: 97 active+clean; 208 MiB data, 867 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.8 MiB/s wr, 750 op/s 2024-05-08T06:36:30.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 31276 18.12 MB/sec warmup 51 sec latency 24.813 ms 2024-05-08T06:36:31.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 32032 18.34 MB/sec warmup 52 sec latency 39.459 ms 2024-05-08T06:36:32.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 32560 18.26 MB/sec warmup 53 sec latency 23.697 ms 2024-05-08T06:36:32.946 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:32 smithi082 ceph-mon[22957]: pgmap v265: 97 pgs: 97 active+clean; 213 MiB data, 916 MiB used, 714 GiB / 715 GiB avail; 8.9 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-05-08T06:36:33.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:32 smithi134 ceph-mon[25063]: pgmap v265: 97 pgs: 97 active+clean; 213 MiB data, 916 MiB used, 714 GiB / 715 GiB avail; 8.9 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-05-08T06:36:33.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 33213 18.07 MB/sec warmup 54 sec latency 22.095 ms 2024-05-08T06:36:34.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 34003 18.15 MB/sec warmup 55 sec latency 24.844 ms 2024-05-08T06:36:34.966 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:34 smithi082 ceph-mon[22957]: pgmap v266: 97 pgs: 97 active+clean; 213 MiB data, 918 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.2 MiB/s wr, 714 op/s 2024-05-08T06:36:35.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:34 smithi134 ceph-mon[25063]: pgmap v266: 97 pgs: 97 active+clean; 213 MiB data, 918 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.2 MiB/s wr, 714 op/s 2024-05-08T06:36:35.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 34743 18.16 MB/sec warmup 56 sec latency 34.797 ms 2024-05-08T06:36:36.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 35440 18.30 MB/sec warmup 57 sec latency 24.245 ms 2024-05-08T06:36:36.740 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:36 smithi082 ceph-mon[22957]: pgmap v267: 97 pgs: 97 active+clean; 218 MiB data, 942 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 816 op/s 2024-05-08T06:36:37.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:36 smithi134 ceph-mon[25063]: pgmap v267: 97 pgs: 97 active+clean; 218 MiB data, 942 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 816 op/s 2024-05-08T06:36:37.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 35934 18.29 MB/sec warmup 58 sec latency 35.221 ms 2024-05-08T06:36:37.752 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:36:37.752 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:36:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:36:38.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:36:38.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 36318 18.09 MB/sec warmup 59 sec latency 42.691 ms 2024-05-08T06:36:38.761 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:38 smithi082 ceph-mon[22957]: pgmap v268: 97 pgs: 97 active+clean; 227 MiB data, 1023 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.09k op/s 2024-05-08T06:36:39.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:38 smithi134 ceph-mon[25063]: pgmap v268: 97 pgs: 97 active+clean; 227 MiB data, 1023 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.09k op/s 2024-05-08T06:36:39.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 36907 17.95 MB/sec warmup 60 sec latency 33.135 ms 2024-05-08T06:36:40.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 37510 17.93 MB/sec warmup 61 sec latency 42.107 ms 2024-05-08T06:36:40.779 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:40 smithi082 ceph-mon[22957]: pgmap v269: 97 pgs: 97 active+clean; 227 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.8 MiB/s wr, 756 op/s 2024-05-08T06:36:41.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:40 smithi134 ceph-mon[25063]: pgmap v269: 97 pgs: 97 active+clean; 227 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.8 MiB/s wr, 756 op/s 2024-05-08T06:36:41.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 38340 17.96 MB/sec warmup 62 sec latency 34.394 ms 2024-05-08T06:36:42.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 39092 18.13 MB/sec warmup 63 sec latency 23.287 ms 2024-05-08T06:36:42.794 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:42 smithi082 ceph-mon[22957]: pgmap v270: 97 pgs: 97 active+clean; 230 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2024-05-08T06:36:43.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:42 smithi134 ceph-mon[25063]: pgmap v270: 97 pgs: 97 active+clean; 230 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2024-05-08T06:36:43.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 39638 18.10 MB/sec warmup 64 sec latency 23.180 ms 2024-05-08T06:36:44.427 INFO:teuthology.orchestra.run.smithi082.stdout: 5 40229 17.94 MB/sec warmup 65 sec latency 91.773 ms 2024-05-08T06:36:44.888 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:44 smithi082 ceph-mon[22957]: pgmap v271: 97 pgs: 97 active+clean; 234 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.7 MiB/s wr, 779 op/s 2024-05-08T06:36:45.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:44 smithi134 ceph-mon[25063]: pgmap v271: 97 pgs: 97 active+clean; 234 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.7 MiB/s wr, 779 op/s 2024-05-08T06:36:45.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 40885 17.96 MB/sec warmup 66 sec latency 60.060 ms 2024-05-08T06:36:46.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 41764 18.01 MB/sec warmup 67 sec latency 29.669 ms 2024-05-08T06:36:46.900 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:46 smithi082 ceph-mon[22957]: pgmap v272: 97 pgs: 97 active+clean; 235 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 800 op/s 2024-05-08T06:36:47.027 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:46 smithi134 ceph-mon[25063]: pgmap v272: 97 pgs: 97 active+clean; 235 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 800 op/s 2024-05-08T06:36:47.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 42387 18.10 MB/sec warmup 68 sec latency 37.373 ms 2024-05-08T06:36:48.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 43010 18.13 MB/sec warmup 69 sec latency 34.528 ms 2024-05-08T06:36:48.937 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:48 smithi082 ceph-mon[22957]: pgmap v273: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 901 op/s 2024-05-08T06:36:49.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:48 smithi134 ceph-mon[25063]: pgmap v273: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 901 op/s 2024-05-08T06:36:49.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 43441 17.99 MB/sec warmup 70 sec latency 29.248 ms 2024-05-08T06:36:50.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 43923 17.84 MB/sec warmup 71 sec latency 34.889 ms 2024-05-08T06:36:50.984 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:50 smithi082 ceph-mon[22957]: pgmap v274: 97 pgs: 97 active+clean; 248 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.3 MiB/s wr, 695 op/s 2024-05-08T06:36:51.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:50 smithi134 ceph-mon[25063]: pgmap v274: 97 pgs: 97 active+clean; 248 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.3 MiB/s wr, 695 op/s 2024-05-08T06:36:51.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 44311 17.71 MB/sec warmup 72 sec latency 73.004 ms 2024-05-08T06:36:52.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 44833 17.67 MB/sec warmup 73 sec latency 90.814 ms 2024-05-08T06:36:52.791 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:52 smithi082 ceph-mon[22957]: pgmap v275: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-05-08T06:36:53.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:52 smithi134 ceph-mon[25063]: pgmap v275: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-05-08T06:36:53.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 45492 17.67 MB/sec warmup 74 sec latency 24.351 ms 2024-05-08T06:36:54.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 46139 17.74 MB/sec warmup 75 sec latency 27.358 ms 2024-05-08T06:36:54.831 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:54 smithi082 ceph-mon[22957]: pgmap v276: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.0 MiB/s wr, 745 op/s 2024-05-08T06:36:55.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:54 smithi134 ceph-mon[25063]: pgmap v276: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.0 MiB/s wr, 745 op/s 2024-05-08T06:36:55.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 46711 17.75 MB/sec warmup 76 sec latency 30.542 ms 2024-05-08T06:36:56.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 47351 17.66 MB/sec warmup 77 sec latency 45.873 ms 2024-05-08T06:36:56.862 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:56 smithi082 ceph-mon[22957]: pgmap v277: 97 pgs: 97 active+clean; 260 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 671 op/s 2024-05-08T06:36:57.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:56 smithi134 ceph-mon[25063]: pgmap v277: 97 pgs: 97 active+clean; 260 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 671 op/s 2024-05-08T06:36:57.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 47975 17.64 MB/sec warmup 78 sec latency 28.925 ms 2024-05-08T06:36:57.871 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:36:57 smithi082 ceph-mon[22957]: pgmap v278: 97 pgs: 97 active+clean; 270 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 955 op/s 2024-05-08T06:36:58.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:36:57 smithi134 ceph-mon[25063]: pgmap v278: 97 pgs: 97 active+clean; 270 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 955 op/s 2024-05-08T06:36:58.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 48768 17.65 MB/sec warmup 79 sec latency 27.076 ms 2024-05-08T06:36:59.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 49317 17.68 MB/sec warmup 80 sec latency 34.303 ms 2024-05-08T06:37:00.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 50019 17.79 MB/sec warmup 81 sec latency 50.641 ms 2024-05-08T06:37:00.674 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:00 smithi082 ceph-mon[22957]: pgmap v279: 97 pgs: 97 active+clean; 270 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.7 MiB/s wr, 753 op/s 2024-05-08T06:37:00.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:00 smithi134 ceph-mon[25063]: pgmap v279: 97 pgs: 97 active+clean; 270 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.7 MiB/s wr, 753 op/s 2024-05-08T06:37:01.430 INFO:teuthology.orchestra.run.smithi082.stdout: 5 50569 17.69 MB/sec warmup 82 sec latency 27.324 ms 2024-05-08T06:37:02.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 51379 17.68 MB/sec warmup 83 sec latency 21.260 ms 2024-05-08T06:37:02.936 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:02 smithi082 ceph-mon[22957]: pgmap v280: 97 pgs: 97 active+clean; 281 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-05-08T06:37:03.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:02 smithi134 ceph-mon[25063]: pgmap v280: 97 pgs: 97 active+clean; 281 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-05-08T06:37:03.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 52361 17.77 MB/sec warmup 84 sec latency 21.095 ms 2024-05-08T06:37:03.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:03 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:37:03.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:03 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:37:03.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:03 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:37:03.942 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:03 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-08T06:37:03.942 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:03 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-08T06:37:03.942 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:03 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-08T06:37:04.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 53108 17.87 MB/sec warmup 85 sec latency 22.102 ms 2024-05-08T06:37:04.950 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:04 smithi082 ceph-mon[22957]: pgmap v281: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.9 MiB/s wr, 770 op/s 2024-05-08T06:37:04.950 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:37:04.950 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:04 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:37:05.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:04 smithi134 ceph-mon[25063]: pgmap v281: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.9 MiB/s wr, 770 op/s 2024-05-08T06:37:05.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:37:05.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:04 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' 2024-05-08T06:37:05.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 53858 17.95 MB/sec warmup 86 sec latency 55.819 ms 2024-05-08T06:37:06.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 54516 17.85 MB/sec warmup 87 sec latency 21.479 ms 2024-05-08T06:37:06.875 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:06 smithi082 ceph-mon[22957]: pgmap v282: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.5 MiB/s wr, 795 op/s 2024-05-08T06:37:07.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:06 smithi134 ceph-mon[25063]: pgmap v282: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.5 MiB/s wr, 795 op/s 2024-05-08T06:37:07.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 55317 17.91 MB/sec warmup 88 sec latency 22.444 ms 2024-05-08T06:37:08.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 56130 17.90 MB/sec warmup 89 sec latency 31.581 ms 2024-05-08T06:37:08.884 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:08 smithi082 ceph-mon[22957]: pgmap v283: 97 pgs: 97 active+clean; 294 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.4 KiB/s rd, 12 MiB/s wr, 1.08k op/s 2024-05-08T06:37:09.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:08 smithi134 ceph-mon[25063]: pgmap v283: 97 pgs: 97 active+clean; 294 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.4 KiB/s rd, 12 MiB/s wr, 1.08k op/s 2024-05-08T06:37:09.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 56933 18.02 MB/sec warmup 90 sec latency 19.355 ms 2024-05-08T06:37:10.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 57759 18.05 MB/sec warmup 91 sec latency 19.796 ms 2024-05-08T06:37:10.892 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:10 smithi082 ceph-mon[22957]: pgmap v284: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.5 MiB/s wr, 794 op/s 2024-05-08T06:37:11.027 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:10 smithi134 ceph-mon[25063]: pgmap v284: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.5 MiB/s wr, 794 op/s 2024-05-08T06:37:11.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 58661 18.08 MB/sec warmup 92 sec latency 16.076 ms 2024-05-08T06:37:12.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 59518 18.12 MB/sec warmup 93 sec latency 25.566 ms 2024-05-08T06:37:12.916 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:12 smithi082 ceph-mon[22957]: pgmap v285: 97 pgs: 97 active+clean; 306 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 13 MiB/s wr, 1.23k op/s 2024-05-08T06:37:13.027 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:12 smithi134 ceph-mon[25063]: pgmap v285: 97 pgs: 97 active+clean; 306 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 13 MiB/s wr, 1.23k op/s 2024-05-08T06:37:13.428 INFO:teuthology.orchestra.run.smithi082.stdout: 5 60287 18.22 MB/sec warmup 94 sec latency 19.396 ms 2024-05-08T06:37:14.429 INFO:teuthology.orchestra.run.smithi082.stdout: 5 60964 18.26 MB/sec warmup 95 sec latency 22.234 ms 2024-05-08T06:37:14.931 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:14 smithi082 ceph-mon[22957]: pgmap v286: 97 pgs: 97 active+clean; 306 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 9.1 MiB/s wr, 860 op/s 2024-05-08T06:37:15.027 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:14 smithi134 ceph-mon[25063]: pgmap v286: 97 pgs: 97 active+clean; 306 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 9.1 MiB/s wr, 860 op/s 2024-05-08T06:37:15.429 INFO:teuthology.orchestra.run.smithi082.stdout: 5 61417 18.12 MB/sec warmup 96 sec latency 40.694 ms 2024-05-08T06:37:16.429 INFO:teuthology.orchestra.run.smithi082.stdout: 5 61920 18.06 MB/sec warmup 97 sec latency 60.653 ms 2024-05-08T06:37:16.969 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:16 smithi082 ceph-mon[22957]: pgmap v287: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.3 MiB/s wr, 815 op/s 2024-05-08T06:37:17.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:16 smithi134 ceph-mon[25063]: pgmap v287: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.3 MiB/s wr, 815 op/s 2024-05-08T06:37:17.429 INFO:teuthology.orchestra.run.smithi082.stdout: 5 62554 18.07 MB/sec warmup 98 sec latency 28.868 ms 2024-05-08T06:37:18.429 INFO:teuthology.orchestra.run.smithi082.stdout: 5 63185 18.02 MB/sec warmup 99 sec latency 32.332 ms 2024-05-08T06:37:19.003 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:18 smithi082 ceph-mon[22957]: pgmap v288: 97 pgs: 97 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 13 MiB/s wr, 1.16k op/s 2024-05-08T06:37:19.027 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:18 smithi134 ceph-mon[25063]: pgmap v288: 97 pgs: 97 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 13 MiB/s wr, 1.16k op/s 2024-05-08T06:37:19.429 INFO:teuthology.orchestra.run.smithi082.stdout: 5 63748 18.06 MB/sec warmup 100 sec latency 27.139 ms 2024-05-08T06:37:20.429 INFO:teuthology.orchestra.run.smithi082.stdout: 5 64416 18.11 MB/sec warmup 101 sec latency 30.648 ms 2024-05-08T06:37:20.761 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:20 smithi082 ceph-mon[22957]: pgmap v289: 97 pgs: 97 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 8.9 MiB/s wr, 852 op/s 2024-05-08T06:37:21.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:20 smithi134 ceph-mon[25063]: pgmap v289: 97 pgs: 97 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 8.9 MiB/s wr, 852 op/s 2024-05-08T06:37:21.429 INFO:teuthology.orchestra.run.smithi082.stdout: 5 64993 17.99 MB/sec warmup 102 sec latency 29.851 ms 2024-05-08T06:37:21.776 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:21 smithi082 ceph-mon[22957]: pgmap v290: 97 pgs: 97 active+clean; 325 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.12k op/s 2024-05-08T06:37:22.027 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:21 smithi134 ceph-mon[25063]: pgmap v290: 97 pgs: 97 active+clean; 325 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.12k op/s 2024-05-08T06:37:22.429 INFO:teuthology.orchestra.run.smithi082.stdout: 5 65619 17.96 MB/sec warmup 103 sec latency 22.254 ms 2024-05-08T06:37:23.429 INFO:teuthology.orchestra.run.smithi082.stdout: 5 66348 17.99 MB/sec warmup 104 sec latency 43.851 ms 2024-05-08T06:37:24.429 INFO:teuthology.orchestra.run.smithi082.stdout: 5 66867 17.95 MB/sec warmup 105 sec latency 22.155 ms 2024-05-08T06:37:24.625 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:24 smithi082 ceph-mon[22957]: pgmap v291: 97 pgs: 97 active+clean; 325 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 7.8 MiB/s wr, 687 op/s 2024-05-08T06:37:24.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:24 smithi134 ceph-mon[25063]: pgmap v291: 97 pgs: 97 active+clean; 325 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 7.8 MiB/s wr, 687 op/s 2024-05-08T06:37:25.429 INFO:teuthology.orchestra.run.smithi082.stdout: 5 67493 17.98 MB/sec warmup 106 sec latency 32.005 ms 2024-05-08T06:37:25.591 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -e sha1=7c8f650b36e258f639fa4a83becade57cbfd2009 -- bash -c 'ceph orch ps' 2024-05-08T06:37:26.429 INFO:teuthology.orchestra.run.smithi082.stdout: 5 67996 17.99 MB/sec warmup 107 sec latency 32.480 ms 2024-05-08T06:37:26.911 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:26 smithi082 ceph-mon[22957]: pgmap v292: 97 pgs: 97 active+clean; 326 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.3 MiB/s wr, 763 op/s 2024-05-08T06:37:27.027 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:26 smithi134 ceph-mon[25063]: pgmap v292: 97 pgs: 97 active+clean; 326 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.3 MiB/s wr, 763 op/s 2024-05-08T06:37:27.429 INFO:teuthology.orchestra.run.smithi082.stdout: 5 68472 17.87 MB/sec warmup 108 sec latency 37.801 ms 2024-05-08T06:37:28.430 INFO:teuthology.orchestra.run.smithi082.stdout: 5 68932 17.80 MB/sec warmup 109 sec latency 62.467 ms 2024-05-08T06:37:28.822 INFO:teuthology.orchestra.run.smithi082.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-08T06:37:28.823 INFO:teuthology.orchestra.run.smithi082.stdout:alertmanager.smithi082 smithi082 *:9093,9094 running (6m) 112s ago 8m 24.4M - ba2b418f427c 86af446dfdba 2024-05-08T06:37:28.823 INFO:teuthology.orchestra.run.smithi082.stdout:crash.smithi082 smithi082 running (8m) 112s ago 8m 6409k - 17.2.0 e1d6a67b021e c5efbe70843e 2024-05-08T06:37:28.823 INFO:teuthology.orchestra.run.smithi082.stdout:crash.smithi134 smithi134 running (7m) 114s ago 7m 6405k - 17.2.0 e1d6a67b021e c670ee9715d5 2024-05-08T06:37:28.823 INFO:teuthology.orchestra.run.smithi082.stdout:grafana.smithi082 smithi082 *:3000 running (6m) 112s ago 8m 56.3M - 8.3.5 dad864ee21e9 903d0b5aab5f 2024-05-08T06:37:28.823 INFO:teuthology.orchestra.run.smithi082.stdout:mds.foofs.smithi082.ofnfgy smithi082 running (2m) 112s ago 2m 13.3M - 17.2.0 e1d6a67b021e 3ccf22c4807b 2024-05-08T06:37:28.823 INFO:teuthology.orchestra.run.smithi082.stdout:mds.foofs.smithi134.ooovgp smithi134 running (2m) 114s ago 2m 12.1M - 17.2.0 e1d6a67b021e a436346080b2 2024-05-08T06:37:28.823 INFO:teuthology.orchestra.run.smithi082.stdout:mgr.smithi082.bsnvtb smithi082 *:9283 running (9m) 112s ago 9m 460M - 17.2.0 e1d6a67b021e affb5f53b21e 2024-05-08T06:37:28.823 INFO:teuthology.orchestra.run.smithi082.stdout:mgr.smithi134.zbsjta smithi134 *:8443,9283 running (7m) 114s ago 7m 413M - 17.2.0 e1d6a67b021e 971c6830c247 2024-05-08T06:37:28.823 INFO:teuthology.orchestra.run.smithi082.stdout:mon.smithi082 smithi082 running (9m) 112s ago 9m 46.5M 2048M 17.2.0 e1d6a67b021e ee82d4cf1e0f 2024-05-08T06:37:28.823 INFO:teuthology.orchestra.run.smithi082.stdout:mon.smithi134 smithi134 running (7m) 114s ago 7m 37.0M 2048M 17.2.0 e1d6a67b021e 302cfd0d2bb4 2024-05-08T06:37:28.823 INFO:teuthology.orchestra.run.smithi082.stdout:nfs.foo.0.0.smithi082.dfhwyg smithi082 *:2049 running (2m) 112s ago 2m 53.1M - 4.0 e1d6a67b021e 596537e6bfac 2024-05-08T06:37:28.823 INFO:teuthology.orchestra.run.smithi082.stdout:nfs.foo.1.0.smithi134.xhqvcd smithi134 *:2049 running (117s) 114s ago 116s 50.8M - 4.0 e1d6a67b021e 13ef5875be00 2024-05-08T06:37:28.823 INFO:teuthology.orchestra.run.smithi082.stdout:node-exporter.smithi082 smithi082 *:9100 running (7m) 112s ago 7m 16.5M - 1dbe0e931976 3ba57430c975 2024-05-08T06:37:28.823 INFO:teuthology.orchestra.run.smithi082.stdout:node-exporter.smithi134 smithi134 *:9100 running (7m) 114s ago 7m 17.0M - 1dbe0e931976 aea98e24f85a 2024-05-08T06:37:28.824 INFO:teuthology.orchestra.run.smithi082.stdout:osd.0 smithi082 running (6m) 112s ago 6m 45.5M 3216M 17.2.0 e1d6a67b021e 4061f89d770d 2024-05-08T06:37:28.824 INFO:teuthology.orchestra.run.smithi082.stdout:osd.1 smithi082 running (6m) 112s ago 6m 45.7M 3216M 17.2.0 e1d6a67b021e 2c86afbd7dc1 2024-05-08T06:37:28.824 INFO:teuthology.orchestra.run.smithi082.stdout:osd.2 smithi082 running (5m) 112s ago 5m 40.0M 3216M 17.2.0 e1d6a67b021e a703fa1e14f7 2024-05-08T06:37:28.824 INFO:teuthology.orchestra.run.smithi082.stdout:osd.3 smithi082 running (5m) 112s ago 5m 40.8M 3216M 17.2.0 e1d6a67b021e 7f3b8f015f59 2024-05-08T06:37:28.824 INFO:teuthology.orchestra.run.smithi082.stdout:osd.4 smithi134 running (4m) 114s ago 4m 41.3M 3984M 17.2.0 e1d6a67b021e 3de640027e4e 2024-05-08T06:37:28.824 INFO:teuthology.orchestra.run.smithi082.stdout:osd.5 smithi134 running (4m) 114s ago 4m 43.9M 3984M 17.2.0 e1d6a67b021e 75f363c21b07 2024-05-08T06:37:28.824 INFO:teuthology.orchestra.run.smithi082.stdout:osd.6 smithi134 running (4m) 114s ago 4m 38.0M 3984M 17.2.0 e1d6a67b021e 98bcf9455cd6 2024-05-08T06:37:28.824 INFO:teuthology.orchestra.run.smithi082.stdout:osd.7 smithi134 running (3m) 114s ago 3m 41.9M 3984M 17.2.0 e1d6a67b021e 9b1f288b1e8c 2024-05-08T06:37:28.824 INFO:teuthology.orchestra.run.smithi082.stdout:prometheus.smithi082 smithi082 *:9095 running (7m) 112s ago 7m 63.0M - 514e6a882f6e 637ca038131b 2024-05-08T06:37:28.925 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:28 smithi082 ceph-mon[22957]: pgmap v293: 97 pgs: 97 active+clean; 336 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 11 MiB/s wr, 1.08k op/s 2024-05-08T06:37:29.027 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:28 smithi134 ceph-mon[25063]: pgmap v293: 97 pgs: 97 active+clean; 336 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 11 MiB/s wr, 1.08k op/s 2024-05-08T06:37:29.429 INFO:teuthology.orchestra.run.smithi082.stdout: 5 69510 17.80 MB/sec warmup 110 sec latency 29.564 ms 2024-05-08T06:37:30.281 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -e sha1=7c8f650b36e258f639fa4a83becade57cbfd2009 -- bash -c 'ceph orch upgrade status' 2024-05-08T06:37:30.429 INFO:teuthology.orchestra.run.smithi082.stdout: 5 70083 17.75 MB/sec warmup 111 sec latency 37.527 ms 2024-05-08T06:37:30.949 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:30 smithi082 ceph-mon[22957]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:37:30.949 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:30 smithi082 ceph-mon[22957]: pgmap v294: 97 pgs: 97 active+clean; 336 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 7.5 MiB/s wr, 721 op/s 2024-05-08T06:37:31.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:30 smithi134 ceph-mon[25063]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:37:31.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:30 smithi134 ceph-mon[25063]: pgmap v294: 97 pgs: 97 active+clean; 336 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 7.5 MiB/s wr, 721 op/s 2024-05-08T06:37:31.429 INFO:teuthology.orchestra.run.smithi082.stdout: 5 70438 17.69 MB/sec warmup 112 sec latency 36.739 ms 2024-05-08T06:37:32.429 INFO:teuthology.orchestra.run.smithi082.stdout: 5 71056 17.72 MB/sec warmup 113 sec latency 57.639 ms 2024-05-08T06:37:32.989 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:32 smithi082 ceph-mon[22957]: pgmap v295: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 10 MiB/s wr, 1.07k op/s 2024-05-08T06:37:33.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:32 smithi134 ceph-mon[25063]: pgmap v295: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 10 MiB/s wr, 1.07k op/s 2024-05-08T06:37:33.431 INFO:teuthology.orchestra.run.smithi082.stdout: 5 71626 17.74 MB/sec warmup 114 sec latency 36.124 ms 2024-05-08T06:37:34.431 INFO:teuthology.orchestra.run.smithi082.stdout: 5 72245 17.66 MB/sec warmup 115 sec latency 26.765 ms 2024-05-08T06:37:34.756 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:34 smithi082 ceph-mon[22957]: pgmap v296: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.1 MiB/s wr, 830 op/s 2024-05-08T06:37:34.768 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2024-05-08T06:37:34.768 INFO:teuthology.orchestra.run.smithi082.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:7c8f650b36e258f639fa4a83becade57cbfd2009", 2024-05-08T06:37:34.768 INFO:teuthology.orchestra.run.smithi082.stdout: "in_progress": true, 2024-05-08T06:37:34.769 INFO:teuthology.orchestra.run.smithi082.stdout: "services_complete": [], 2024-05-08T06:37:34.769 INFO:teuthology.orchestra.run.smithi082.stdout: "progress": "", 2024-05-08T06:37:34.769 INFO:teuthology.orchestra.run.smithi082.stdout: "message": "Error: UPGRADE_FAILED_PULL: Upgrade: failed to pull target image" 2024-05-08T06:37:34.769 INFO:teuthology.orchestra.run.smithi082.stdout:} 2024-05-08T06:37:35.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:34 smithi134 ceph-mon[25063]: pgmap v296: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.1 MiB/s wr, 830 op/s 2024-05-08T06:37:35.431 INFO:teuthology.orchestra.run.smithi082.stdout: 5 72968 17.70 MB/sec warmup 116 sec latency 18.977 ms 2024-05-08T06:37:35.660 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -e sha1=7c8f650b36e258f639fa4a83becade57cbfd2009 -- bash -c 'ceph health detail' 2024-05-08T06:37:36.431 INFO:teuthology.orchestra.run.smithi082.stdout: 5 73737 17.67 MB/sec warmup 117 sec latency 27.107 ms 2024-05-08T06:37:36.773 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:36 smithi082 ceph-mon[22957]: from='client.14632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:37:36.774 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:36 smithi082 ceph-mon[22957]: pgmap v297: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.5 MiB/s wr, 844 op/s 2024-05-08T06:37:37.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:36 smithi134 ceph-mon[25063]: from='client.14632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-08T06:37:37.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:36 smithi134 ceph-mon[25063]: pgmap v297: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.5 MiB/s wr, 844 op/s 2024-05-08T06:37:37.431 INFO:teuthology.orchestra.run.smithi082.stdout: 5 74332 17.71 MB/sec warmup 118 sec latency 47.798 ms 2024-05-08T06:37:37.792 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:37:37.792 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:37 smithi082 ceph-mon[22957]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:37:38.027 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/mirror_snapshot_schedule"}]: dispatch 2024-05-08T06:37:38.027 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:37 smithi134 ceph-mon[25063]: from='mgr.14200 172.21.15.82:0/3108382451' entity='mgr.smithi082.bsnvtb' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi082.bsnvtb/trash_purge_schedule"}]: dispatch 2024-05-08T06:37:38.431 INFO:teuthology.orchestra.run.smithi082.stdout: 5 74801 17.68 MB/sec warmup 119 sec latency 37.338 ms 2024-05-08T06:37:38.801 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:38 smithi082 ceph-mon[22957]: pgmap v298: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 9.8 MiB/s wr, 1.01k op/s 2024-05-08T06:37:39.027 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:38 smithi134 ceph-mon[25063]: pgmap v298: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 9.8 MiB/s wr, 1.01k op/s 2024-05-08T06:37:39.371 INFO:teuthology.orchestra.run.smithi082.stdout:HEALTH_WARN Upgrade: failed to pull target image 2024-05-08T06:37:39.371 INFO:teuthology.orchestra.run.smithi082.stdout:[WRN] UPGRADE_FAILED_PULL: Upgrade: failed to pull target image 2024-05-08T06:37:39.371 INFO:teuthology.orchestra.run.smithi082.stdout: host smithi082 `cephadm pull` failed: cephadm exited with an error code: 1, stderr: Pulling container image quay.ceph.io/ceph-ci/ceph:7c8f650b36e258f639fa4a83becade57cbfd2009... 2024-05-08T06:37:39.371 INFO:teuthology.orchestra.run.smithi082.stdout:Non-zero exit code 125 from /bin/podman pull quay.ceph.io/ceph-ci/ceph:7c8f650b36e258f639fa4a83becade57cbfd2009 2024-05-08T06:37:39.371 INFO:teuthology.orchestra.run.smithi082.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:7c8f650b36e258f639fa4a83becade57cbfd2009... 2024-05-08T06:37:39.372 INFO:teuthology.orchestra.run.smithi082.stdout:/bin/podman: stderr Error: initializing source docker://quay.ceph.io/ceph-ci/ceph:7c8f650b36e258f639fa4a83becade57cbfd2009: reading manifest 7c8f650b36e258f639fa4a83becade57cbfd2009 in quay.ceph.io/ceph-ci/ceph: manifest unknown 2024-05-08T06:37:39.372 INFO:teuthology.orchestra.run.smithi082.stdout:ERROR: Failed command: /bin/podman pull quay.ceph.io/ceph-ci/ceph:7c8f650b36e258f639fa4a83becade57cbfd2009 2024-05-08T06:37:39.831 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:39 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3031260665' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-08T06:37:40.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:39 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/3031260665' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-08T06:37:40.386 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -e sha1=7c8f650b36e258f639fa4a83becade57cbfd2009 -- bash -c 'ceph versions' 2024-05-08T06:37:40.431 INFO:teuthology.orchestra.run.smithi082.stdout: 5 76037 12.03 MB/sec execute 1 sec latency 26.838 ms 2024-05-08T06:37:40.844 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:40 smithi082 ceph-mon[22957]: pgmap v299: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 6.6 MiB/s wr, 702 op/s 2024-05-08T06:37:41.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:40 smithi134 ceph-mon[25063]: pgmap v299: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 6.6 MiB/s wr, 702 op/s 2024-05-08T06:37:41.431 INFO:teuthology.orchestra.run.smithi082.stdout: 5 76666 16.65 MB/sec execute 2 sec latency 23.350 ms 2024-05-08T06:37:41.855 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:41 smithi082 ceph-mon[22957]: pgmap v300: 97 pgs: 97 active+clean; 338 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 10 MiB/s wr, 967 op/s 2024-05-08T06:37:42.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:41 smithi134 ceph-mon[25063]: pgmap v300: 97 pgs: 97 active+clean; 338 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 10 MiB/s wr, 967 op/s 2024-05-08T06:37:42.431 INFO:teuthology.orchestra.run.smithi082.stdout: 5 77336 15.11 MB/sec execute 3 sec latency 39.675 ms 2024-05-08T06:37:43.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 77901 16.57 MB/sec execute 4 sec latency 44.629 ms 2024-05-08T06:37:43.903 INFO:teuthology.orchestra.run.smithi082.stdout:{ 2024-05-08T06:37:43.903 INFO:teuthology.orchestra.run.smithi082.stdout: "mon": { 2024-05-08T06:37:43.903 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-08T06:37:43.903 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2024-05-08T06:37:43.904 INFO:teuthology.orchestra.run.smithi082.stdout: "mgr": { 2024-05-08T06:37:43.904 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-08T06:37:43.904 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2024-05-08T06:37:43.904 INFO:teuthology.orchestra.run.smithi082.stdout: "osd": { 2024-05-08T06:37:43.904 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-08T06:37:43.904 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2024-05-08T06:37:43.904 INFO:teuthology.orchestra.run.smithi082.stdout: "mds": { 2024-05-08T06:37:43.904 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-08T06:37:43.904 INFO:teuthology.orchestra.run.smithi082.stdout: }, 2024-05-08T06:37:43.904 INFO:teuthology.orchestra.run.smithi082.stdout: "overall": { 2024-05-08T06:37:43.904 INFO:teuthology.orchestra.run.smithi082.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-05-08T06:37:43.904 INFO:teuthology.orchestra.run.smithi082.stdout: } 2024-05-08T06:37:43.904 INFO:teuthology.orchestra.run.smithi082.stdout:} 2024-05-08T06:37:44.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 78522 17.84 MB/sec execute 5 sec latency 23.154 ms 2024-05-08T06:37:44.616 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:44 smithi082 ceph-mon[22957]: pgmap v301: 97 pgs: 97 active+clean; 342 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 7.5 MiB/s wr, 699 op/s 2024-05-08T06:37:44.616 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:44 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/176515157' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-08T06:37:44.718 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -e sha1=7c8f650b36e258f639fa4a83becade57cbfd2009 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-05-08T06:37:44.777 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:44 smithi134 ceph-mon[25063]: pgmap v301: 97 pgs: 97 active+clean; 342 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 7.5 MiB/s wr, 699 op/s 2024-05-08T06:37:44.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:44 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/176515157' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-08T06:37:45.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 78954 15.99 MB/sec execute 6 sec latency 39.123 ms 2024-05-08T06:37:46.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 79394 14.75 MB/sec execute 7 sec latency 39.677 ms 2024-05-08T06:37:46.911 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:46 smithi082 ceph-mon[22957]: pgmap v302: 97 pgs: 97 active+clean; 342 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 668 op/s 2024-05-08T06:37:47.027 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:46 smithi134 ceph-mon[25063]: pgmap v302: 97 pgs: 97 active+clean; 342 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 668 op/s 2024-05-08T06:37:47.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 79961 14.55 MB/sec execute 8 sec latency 22.786 ms 2024-05-08T06:37:48.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 80576 15.04 MB/sec execute 9 sec latency 46.316 ms 2024-05-08T06:37:48.466 INFO:teuthology.orchestra.run.smithi082.stdout:true 2024-05-08T06:37:48.920 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:48 smithi082 ceph-mon[22957]: pgmap v303: 97 pgs: 97 active+clean; 335 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 949 op/s 2024-05-08T06:37:48.920 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:48 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3877500321' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-08T06:37:49.027 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:48 smithi134 ceph-mon[25063]: pgmap v303: 97 pgs: 97 active+clean; 335 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 949 op/s 2024-05-08T06:37:49.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:48 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/3877500321' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-08T06:37:49.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 81115 14.78 MB/sec execute 10 sec latency 37.602 ms 2024-05-08T06:37:49.577 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -e sha1=7c8f650b36e258f639fa4a83becade57cbfd2009 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-05-08T06:37:50.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 81668 15.26 MB/sec execute 11 sec latency 32.879 ms 2024-05-08T06:37:50.940 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:50 smithi082 ceph-mon[22957]: pgmap v304: 97 pgs: 97 active+clean; 335 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 7.4 MiB/s wr, 703 op/s 2024-05-08T06:37:51.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:50 smithi134 ceph-mon[25063]: pgmap v304: 97 pgs: 97 active+clean; 335 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 7.4 MiB/s wr, 703 op/s 2024-05-08T06:37:51.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 82190 15.49 MB/sec execute 12 sec latency 34.591 ms 2024-05-08T06:37:52.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 82576 14.87 MB/sec execute 13 sec latency 30.324 ms 2024-05-08T06:37:52.978 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:52 smithi082 ceph-mon[22957]: pgmap v305: 97 pgs: 97 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 11 MiB/s wr, 992 op/s 2024-05-08T06:37:52.978 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:52 smithi082 ceph-mon[22957]: from='client.? 172.21.15.82:0/3215835428' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-08T06:37:53.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:52 smithi134 ceph-mon[25063]: pgmap v305: 97 pgs: 97 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 11 MiB/s wr, 992 op/s 2024-05-08T06:37:53.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:52 smithi134 ceph-mon[25063]: from='client.? 172.21.15.82:0/3215835428' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-08T06:37:53.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 83066 14.47 MB/sec execute 14 sec latency 50.876 ms 2024-05-08T06:37:53.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-08T06:37:53.919 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_7c8f650b36e258f639fa4a83becade57cbfd2009/qa/tasks/cephadm.py", line 1482, in shell _shell( File "/home/teuthworker/src/git.ceph.com_ceph-c_7c8f650b36e258f639fa4a83becade57cbfd2009/qa/tasks/cephadm.py", line 110, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi082 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -e sha1=7c8f650b36e258f639fa4a83becade57cbfd2009 -- bash -c \'ceph versions | jq -e \'"\'"\'.overall | keys\'"\'"\' | grep $sha1\'' 2024-05-08T06:37:54.125 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=6c9bfa2f067a4777baad0ad056dd31e4 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_7c8f650b36e258f639fa4a83becade57cbfd2009/qa/tasks/cephadm.py", line 1482, in shell _shell( File "/home/teuthworker/src/git.ceph.com_ceph-c_7c8f650b36e258f639fa4a83becade57cbfd2009/qa/tasks/cephadm.py", line 110, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_1ae7ad82388e92a475afff437d49054826c019a1/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi082 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -e sha1=7c8f650b36e258f639fa4a83becade57cbfd2009 -- bash -c \'ceph versions | jq -e \'"\'"\'.overall | keys\'"\'"\' | grep $sha1\'' 2024-05-08T06:37:54.129 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-05-08T06:37:54.138 INFO:tasks.cephadm:Teardown begin 2024-05-08T06:37:54.139 DEBUG:teuthology.orchestra.run.smithi082:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-08T06:37:54.179 DEBUG:teuthology.orchestra.run.smithi134:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-08T06:37:54.209 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-05-08T06:37:54.209 DEBUG:teuthology.orchestra.run.smithi082:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-05-08T06:37:54.239 DEBUG:teuthology.orchestra.run.smithi134:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-05-08T06:37:54.266 INFO:tasks.cephadm:Stopping all daemons... 2024-05-08T06:37:54.267 INFO:tasks.cephadm.mon.smithi082:Stopping mon.smithi082... 2024-05-08T06:37:54.267 DEBUG:teuthology.orchestra.run.smithi082:> sudo systemctl stop ceph-02181fb0-0d04-11ef-bc97-c7b262605968@mon.smithi082 2024-05-08T06:37:54.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 83492 14.13 MB/sec execute 15 sec latency 27.743 ms 2024-05-08T06:37:54.496 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:54 smithi082 systemd[1]: Stopping Ceph mon.smithi082 for 02181fb0-0d04-11ef-bc97-c7b262605968... 2024-05-08T06:37:54.496 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:54 smithi082 bash[55914]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon.smithi082" found: no such container 2024-05-08T06:37:54.751 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:54 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi082[22952]: 2024-05-08T06:37:54.533+0000 7fa5fec5b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi082 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-08T06:37:54.751 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:54 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi082[22952]: 2024-05-08T06:37:54.533+0000 7fa5fec5b700 -1 mon.smithi082@0(leader) e2 *** Got Signal Terminated *** 2024-05-08T06:37:54.751 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:54 smithi082 podman[55925]: 2024-05-08 06:37:54.557189597 +0000 UTC m=+0.164343461 container died ee82d4cf1e0f1a57ca19a0926527d64c21ad44178699e3fd6802ba5508a22543 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi082, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, name=centos-stream, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, vcs-type=git, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, release=754, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-05-08T06:37:55.254 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:55 smithi082 podman[55925]: 2024-05-08 06:37:55.180467268 +0000 UTC m=+0.787621126 container cleanup ee82d4cf1e0f1a57ca19a0926527d64c21ad44178699e3fd6802ba5508a22543 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi082, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, name=centos-stream, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-05-08T06:37:55.254 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:55 smithi082 bash[55925]: ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi082 2024-05-08T06:37:55.254 INFO:journalctl@ceph.mon.smithi082.smithi082.stdout:May 08 06:37:55 smithi082 bash[55949]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon.smithi082" found: no such container 2024-05-08T06:37:55.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84004 14.39 MB/sec execute 16 sec latency 34.076 ms 2024-05-08T06:37:55.595 DEBUG:teuthology.orchestra.run.smithi082:> sudo pkill -f 'journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@mon.smithi082.service' 2024-05-08T06:37:55.718 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-08T06:37:55.718 INFO:tasks.cephadm.mon.smithi082:Stopped mon.smithi082 2024-05-08T06:37:55.718 INFO:tasks.cephadm.mon.smithi134:Stopping mon.smithi134... 2024-05-08T06:37:55.718 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-02181fb0-0d04-11ef-bc97-c7b262605968@mon.smithi134 2024-05-08T06:37:56.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:55 smithi134 systemd[1]: Stopping Ceph mon.smithi134 for 02181fb0-0d04-11ef-bc97-c7b262605968... 2024-05-08T06:37:56.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:55 smithi134 bash[41186]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon.smithi134" found: no such container 2024-05-08T06:37:56.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:55 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi134[25045]: 2024-05-08T06:37:55.928+0000 7f33c5bea700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi134 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-08T06:37:56.028 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:55 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi134[25045]: 2024-05-08T06:37:55.928+0000 7f33c5bea700 -1 mon.smithi134@1(peon) e2 *** Got Signal Terminated *** 2024-05-08T06:37:56.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 13.89 MB/sec execute 17 sec latency 456.753 ms 2024-05-08T06:37:56.461 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:56 smithi134 podman[41197]: 2024-05-08 06:37:56.153390843 +0000 UTC m=+0.350313611 container died 302cfd0d2bb4416dcac4ad7d5672305f76737832e63f46a3d38e9a9cf3905f72 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi134, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, io.openshift.tags=base centos centos-stream, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_CLEAN=True, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, release=754, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-05-08T06:37:56.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:56 smithi134 podman[41197]: 2024-05-08 06:37:56.62134839 +0000 UTC m=+0.818271146 container cleanup 302cfd0d2bb4416dcac4ad7d5672305f76737832e63f46a3d38e9a9cf3905f72 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi134, io.openshift.tags=base centos centos-stream, release=754, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, version=8, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True) 2024-05-08T06:37:56.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:56 smithi134 bash[41197]: ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon-smithi134 2024-05-08T06:37:56.778 INFO:journalctl@ceph.mon.smithi134.smithi134.stdout:May 08 06:37:56 smithi134 bash[41223]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-mon.smithi134" found: no such container 2024-05-08T06:37:56.918 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@mon.smithi134.service' 2024-05-08T06:37:56.976 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-08T06:37:56.976 INFO:tasks.cephadm.mon.smithi134:Stopped mon.smithi134 2024-05-08T06:37:56.977 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-05-08T06:37:56.977 DEBUG:teuthology.orchestra.run.smithi082:> sudo systemctl stop ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.0 2024-05-08T06:37:57.417 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:37:57 smithi082 systemd[1]: Stopping Ceph osd.0 for 02181fb0-0d04-11ef-bc97-c7b262605968... 2024-05-08T06:37:57.417 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:37:57 smithi082 bash[56001]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd.0" found: no such container 2024-05-08T06:37:57.417 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:37:57 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-0[33372]: 2024-05-08T06:37:57.205+0000 7f030ac0a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-08T06:37:57.417 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:37:57 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-0[33372]: 2024-05-08T06:37:57.205+0000 7f030ac0a700 -1 osd.0 70 *** Got signal Terminated *** 2024-05-08T06:37:57.418 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:37:57 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-0[33372]: 2024-05-08T06:37:57.205+0000 7f030ac0a700 -1 osd.0 70 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-08T06:37:57.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 13.12 MB/sec execute 18 sec latency 1456.801 ms 2024-05-08T06:37:58.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 12.43 MB/sec execute 19 sec latency 2456.825 ms 2024-05-08T06:37:59.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 11.81 MB/sec execute 20 sec latency 3456.891 ms 2024-05-08T06:38:00.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 11.24 MB/sec execute 21 sec latency 4456.944 ms 2024-05-08T06:38:01.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 10.73 MB/sec execute 22 sec latency 5456.995 ms 2024-05-08T06:38:02.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 10.27 MB/sec execute 23 sec latency 6457.044 ms 2024-05-08T06:38:03.053 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:38:02 smithi082 podman[56014]: 2024-05-08 06:38:02.731409215 +0000 UTC m=+5.650874394 container died 4061f89d770d8e5fdde9ae5a65a2f14dd5fb5a5be3f59b796d0bc1f71c2557c0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-0, release=754, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , ceph=True, distribution-scope=public, io.openshift.expose-services=, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-05-08T06:38:03.352 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:38:03 smithi082 podman[56014]: 2024-05-08 06:38:03.169679524 +0000 UTC m=+6.089144702 container cleanup 4061f89d770d8e5fdde9ae5a65a2f14dd5fb5a5be3f59b796d0bc1f71c2557c0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-0, GIT_BRANCH=HEAD, GIT_CLEAN=True, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, name=centos-stream, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., architecture=x86_64, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, io.buildah.version=1.19.8, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-05-08T06:38:03.352 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:38:03 smithi082 bash[56014]: ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-0 2024-05-08T06:38:03.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 9.84 MB/sec execute 24 sec latency 7457.070 ms 2024-05-08T06:38:03.629 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:38:03 smithi082 podman[56031]: 2024-05-08 06:38:03.354246796 +0000 UTC m=+0.618561334 container remove 4061f89d770d8e5fdde9ae5a65a2f14dd5fb5a5be3f59b796d0bc1f71c2557c0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-0, GIT_CLEAN=True, io.openshift.expose-services=, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, ceph=True, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, version=8, GIT_BRANCH=HEAD) 2024-05-08T06:38:03.630 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:38:03 smithi082 bash[56043]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd.0" found: no such container 2024-05-08T06:38:03.917 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:38:03 smithi082 podman[56109]: 2024-05-08 06:38:03.535388468 +0000 UTC m=+0.018797837 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-08T06:38:03.917 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:38:03 smithi082 podman[56109]: 2024-05-08 06:38:03.788783823 +0000 UTC m=+0.272193169 container create bffa55148a645da3ecdc7e88af4785983f94d1bb9d6115dd3d07208dec687031 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, version=8, vcs-type=git, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754) 2024-05-08T06:38:04.372 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:38:04 smithi082 podman[56109]: 2024-05-08 06:38:04.03304864 +0000 UTC m=+0.516457989 container init bffa55148a645da3ecdc7e88af4785983f94d1bb9d6115dd3d07208dec687031 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-0-deactivate, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, version=8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, ceph=True, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream) 2024-05-08T06:38:04.372 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:38:04 smithi082 podman[56109]: 2024-05-08 06:38:04.038594351 +0000 UTC m=+0.522003700 container start bffa55148a645da3ecdc7e88af4785983f94d1bb9d6115dd3d07208dec687031 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-0-deactivate, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, ceph=True, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, io.openshift.expose-services=, release=754, distribution-scope=public, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , name=centos-stream, version=8, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-05-08T06:38:04.372 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:38:04 smithi082 podman[56109]: 2024-05-08 06:38:04.133124025 +0000 UTC m=+0.616533380 container attach bffa55148a645da3ecdc7e88af4785983f94d1bb9d6115dd3d07208dec687031 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-0-deactivate, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, architecture=x86_64, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.openshift.expose-services=, io.buildah.version=1.19.8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, version=8) 2024-05-08T06:38:04.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 9.45 MB/sec execute 25 sec latency 8457.097 ms 2024-05-08T06:38:04.667 INFO:journalctl@ceph.osd.0.smithi082.stdout:May 08 06:38:04 smithi082 podman[56109]: 2024-05-08 06:38:04.373843333 +0000 UTC m=+0.857252683 container died bffa55148a645da3ecdc7e88af4785983f94d1bb9d6115dd3d07208dec687031 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-0-deactivate, com.redhat.component=centos-stream-container, name=centos-stream, version=8, GIT_CLEAN=True, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, io.buildah.version=1.19.8) 2024-05-08T06:38:05.081 DEBUG:teuthology.orchestra.run.smithi082:> sudo pkill -f 'journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.0.service' 2024-05-08T06:38:05.118 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-08T06:38:05.118 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-05-08T06:38:05.118 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-05-08T06:38:05.119 DEBUG:teuthology.orchestra.run.smithi082:> sudo systemctl stop ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.1 2024-05-08T06:38:05.351 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:38:05 smithi082 systemd[1]: Stopping Ceph osd.1 for 02181fb0-0d04-11ef-bc97-c7b262605968... 2024-05-08T06:38:05.351 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:38:05 smithi082 bash[56186]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd.1" found: no such container 2024-05-08T06:38:05.432 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 9.08 MB/sec execute 26 sec latency 9457.120 ms 2024-05-08T06:38:05.667 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:38:05 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-1[36244]: 2024-05-08T06:38:05.351+0000 7fe27ce9d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-08T06:38:05.667 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:38:05 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-1[36244]: 2024-05-08T06:38:05.351+0000 7fe27ce9d700 -1 osd.1 70 *** Got signal Terminated *** 2024-05-08T06:38:05.667 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:38:05 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-1[36244]: 2024-05-08T06:38:05.351+0000 7fe27ce9d700 -1 osd.1 70 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-08T06:38:06.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 8.75 MB/sec execute 27 sec latency 10457.146 ms 2024-05-08T06:38:07.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 8.43 MB/sec execute 28 sec latency 11457.174 ms 2024-05-08T06:38:08.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 8.14 MB/sec execute 29 sec latency 12457.225 ms 2024-05-08T06:38:09.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 7.87 MB/sec execute 30 sec latency 13457.252 ms 2024-05-08T06:38:10.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 7.62 MB/sec execute 31 sec latency 14457.306 ms 2024-05-08T06:38:11.071 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:38:10 smithi082 podman[56200]: 2024-05-08 06:38:10.789850689 +0000 UTC m=+5.537929194 container died 2c86afbd7dc193ee25b0ca780f955a77cb799c7914ebd8e58cacea27a50c7cdc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-1, vcs-type=git, GIT_BRANCH=HEAD, distribution-scope=public, RELEASE=HEAD, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, name=centos-stream, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, ceph=True, maintainer=Guillaume Abrioux , release=754, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, architecture=x86_64) 2024-05-08T06:38:11.363 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:38:11 smithi082 podman[56200]: 2024-05-08 06:38:11.163453001 +0000 UTC m=+5.911531511 container cleanup 2c86afbd7dc193ee25b0ca780f955a77cb799c7914ebd8e58cacea27a50c7cdc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-1, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, release=754, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, ceph=True, architecture=x86_64) 2024-05-08T06:38:11.363 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:38:11 smithi082 bash[56200]: ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-1 2024-05-08T06:38:11.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 7.38 MB/sec execute 32 sec latency 15457.337 ms 2024-05-08T06:38:11.655 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:38:11 smithi082 podman[56219]: 2024-05-08 06:38:11.364957212 +0000 UTC m=+0.566855594 container remove 2c86afbd7dc193ee25b0ca780f955a77cb799c7914ebd8e58cacea27a50c7cdc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-1, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , version=8, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, architecture=x86_64, GIT_CLEAN=True, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-05-08T06:38:11.656 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:38:11 smithi082 bash[56232]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd.1" found: no such container 2024-05-08T06:38:11.917 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:38:11 smithi082 podman[56296]: 2024-05-08 06:38:11.657414666 +0000 UTC m=+0.103605734 container create 5f99b27359e2873a79d4b24282463907c97ec715a56ac0be2daa7b0ac6eea68e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-1-deactivate, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, ceph=True, io.k8s.display-name=CentOS Stream 8, release=754, GIT_CLEAN=True, vcs-type=git, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-05-08T06:38:11.917 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:38:11 smithi082 podman[56296]: 2024-05-08 06:38:11.572482772 +0000 UTC m=+0.018673840 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-08T06:38:11.917 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:38:11 smithi082 podman[56296]: 2024-05-08 06:38:11.893321809 +0000 UTC m=+0.339512885 container init 5f99b27359e2873a79d4b24282463907c97ec715a56ac0be2daa7b0ac6eea68e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-1-deactivate, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, vcs-type=git, name=centos-stream, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=) 2024-05-08T06:38:11.917 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:38:11 smithi082 podman[56296]: 2024-05-08 06:38:11.900712861 +0000 UTC m=+0.346903933 container start 5f99b27359e2873a79d4b24282463907c97ec715a56ac0be2daa7b0ac6eea68e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-1-deactivate, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, ceph=True, name=centos-stream, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, GIT_BRANCH=HEAD, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, io.openshift.expose-services=, com.redhat.component=centos-stream-container, architecture=x86_64, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-05-08T06:38:12.233 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:38:11 smithi082 podman[56296]: 2024-05-08 06:38:11.976348695 +0000 UTC m=+0.422539773 container attach 5f99b27359e2873a79d4b24282463907c97ec715a56ac0be2daa7b0ac6eea68e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-1-deactivate, RELEASE=HEAD, distribution-scope=public, release=754, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, maintainer=Guillaume Abrioux , version=8, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, ceph=True, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, architecture=x86_64) 2024-05-08T06:38:12.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 7.16 MB/sec execute 33 sec latency 16457.363 ms 2024-05-08T06:38:12.667 INFO:journalctl@ceph.osd.1.smithi082.stdout:May 08 06:38:12 smithi082 podman[56296]: 2024-05-08 06:38:12.235752078 +0000 UTC m=+0.681943142 container died 5f99b27359e2873a79d4b24282463907c97ec715a56ac0be2daa7b0ac6eea68e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, vcs-type=git, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , RELEASE=HEAD, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-05-08T06:38:12.755 DEBUG:teuthology.orchestra.run.smithi082:> sudo pkill -f 'journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.1.service' 2024-05-08T06:38:12.792 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-08T06:38:12.792 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-05-08T06:38:12.792 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-05-08T06:38:12.792 DEBUG:teuthology.orchestra.run.smithi082:> sudo systemctl stop ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.2 2024-05-08T06:38:13.041 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:38:12 smithi082 systemd[1]: Stopping Ceph osd.2 for 02181fb0-0d04-11ef-bc97-c7b262605968... 2024-05-08T06:38:13.041 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:38:12 smithi082 bash[56375]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd.2" found: no such container 2024-05-08T06:38:13.417 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:38:13 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-2[40115]: 2024-05-08T06:38:13.040+0000 7fea186e7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-08T06:38:13.417 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:38:13 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-2[40115]: 2024-05-08T06:38:13.040+0000 7fea186e7700 -1 osd.2 70 *** Got signal Terminated *** 2024-05-08T06:38:13.417 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:38:13 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-2[40115]: 2024-05-08T06:38:13.040+0000 7fea186e7700 -1 osd.2 70 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-08T06:38:13.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 6.95 MB/sec execute 34 sec latency 17457.414 ms 2024-05-08T06:38:14.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 6.75 MB/sec execute 35 sec latency 18457.468 ms 2024-05-08T06:38:15.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 6.56 MB/sec execute 36 sec latency 19457.491 ms 2024-05-08T06:38:16.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 6.38 MB/sec execute 37 sec latency 20457.544 ms 2024-05-08T06:38:17.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 6.21 MB/sec execute 38 sec latency 21457.590 ms 2024-05-08T06:38:18.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 6.05 MB/sec execute 39 sec latency 22457.641 ms 2024-05-08T06:38:18.838 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:38:18 smithi082 podman[56388]: 2024-05-08 06:38:18.495752211 +0000 UTC m=+5.552743809 container died a703fa1e14f7773a7e8b3ffb11d8dee5d83229d942e00cccb6e1102880a546f8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-2, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., io.openshift.expose-services=, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, release=754, build-date=2022-05-03T08:36:31.336870, name=centos-stream, distribution-scope=public, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.k8s.display-name=CentOS Stream 8) 2024-05-08T06:38:18.838 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:38:18 smithi082 podman[56388]: 2024-05-08 06:38:18.839937062 +0000 UTC m=+5.896928663 container cleanup a703fa1e14f7773a7e8b3ffb11d8dee5d83229d942e00cccb6e1102880a546f8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-2, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, RELEASE=HEAD, vendor=Red Hat, Inc., vcs-type=git, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, distribution-scope=public, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-05-08T06:38:19.167 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:38:18 smithi082 bash[56388]: ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-2 2024-05-08T06:38:19.167 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:38:19 smithi082 podman[56410]: 2024-05-08 06:38:19.016210777 +0000 UTC m=+0.516176003 container remove a703fa1e14f7773a7e8b3ffb11d8dee5d83229d942e00cccb6e1102880a546f8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-2, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, version=8, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, release=754, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, ceph=True, io.openshift.expose-services=, com.redhat.component=centos-stream-container) 2024-05-08T06:38:19.167 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:38:19 smithi082 bash[56423]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd.2" found: no such container 2024-05-08T06:38:19.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 5.90 MB/sec execute 40 sec latency 23457.662 ms 2024-05-08T06:38:19.559 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:38:19 smithi082 podman[56485]: 2024-05-08 06:38:19.196757211 +0000 UTC m=+0.018756382 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-08T06:38:19.559 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:38:19 smithi082 podman[56485]: 2024-05-08 06:38:19.317045714 +0000 UTC m=+0.139044873 container create f55b5d5fffd20394ad98bd30c1b80dcacb8cabb4891d111cb50d0c1c251f293e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, distribution-scope=public, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.openshift.tags=base centos centos-stream, architecture=x86_64, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-05-08T06:38:19.559 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:38:19 smithi082 podman[56485]: 2024-05-08 06:38:19.560964037 +0000 UTC m=+0.382963201 container init f55b5d5fffd20394ad98bd30c1b80dcacb8cabb4891d111cb50d0c1c251f293e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-2-deactivate, vcs-type=git, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, vendor=Red Hat, Inc., version=8, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, GIT_CLEAN=True, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, release=754) 2024-05-08T06:38:19.909 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:38:19 smithi082 podman[56485]: 2024-05-08 06:38:19.567696654 +0000 UTC m=+0.389695806 container start f55b5d5fffd20394ad98bd30c1b80dcacb8cabb4891d111cb50d0c1c251f293e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-2-deactivate, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , release=754, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, distribution-scope=public, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, ceph=True, version=8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-05-08T06:38:19.909 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:38:19 smithi082 podman[56485]: 2024-05-08 06:38:19.644236947 +0000 UTC m=+0.466236099 container attach f55b5d5fffd20394ad98bd30c1b80dcacb8cabb4891d111cb50d0c1c251f293e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-2-deactivate, com.redhat.component=centos-stream-container, RELEASE=HEAD, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, version=8, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, vcs-type=git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-05-08T06:38:19.909 INFO:journalctl@ceph.osd.2.smithi082.stdout:May 08 06:38:19 smithi082 podman[56485]: 2024-05-08 06:38:19.910524266 +0000 UTC m=+0.732523431 container died f55b5d5fffd20394ad98bd30c1b80dcacb8cabb4891d111cb50d0c1c251f293e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-2-deactivate, io.openshift.expose-services=, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, distribution-scope=public, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, version=8) 2024-05-08T06:38:20.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 5.76 MB/sec execute 41 sec latency 24457.708 ms 2024-05-08T06:38:20.476 DEBUG:teuthology.orchestra.run.smithi082:> sudo pkill -f 'journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.2.service' 2024-05-08T06:38:20.514 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-08T06:38:20.514 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-05-08T06:38:20.514 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-05-08T06:38:20.515 DEBUG:teuthology.orchestra.run.smithi082:> sudo systemctl stop ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.3 2024-05-08T06:38:20.763 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:38:20 smithi082 systemd[1]: Stopping Ceph osd.3 for 02181fb0-0d04-11ef-bc97-c7b262605968... 2024-05-08T06:38:20.763 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:38:20 smithi082 bash[56563]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd.3" found: no such container 2024-05-08T06:38:21.167 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:38:20 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-3[43962]: 2024-05-08T06:38:20.764+0000 7fda9e141700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-08T06:38:21.167 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:38:20 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-3[43962]: 2024-05-08T06:38:20.764+0000 7fda9e141700 -1 osd.3 70 *** Got signal Terminated *** 2024-05-08T06:38:21.167 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:38:20 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-3[43962]: 2024-05-08T06:38:20.764+0000 7fda9e141700 -1 osd.3 70 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-08T06:38:21.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 5.62 MB/sec execute 42 sec latency 25457.757 ms 2024-05-08T06:38:22.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 5.49 MB/sec execute 43 sec latency 26457.819 ms 2024-05-08T06:38:23.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 5.37 MB/sec execute 44 sec latency 27457.868 ms 2024-05-08T06:38:24.167 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:38:23 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-3[43962]: 2024-05-08T06:38:23.806+0000 7fda9bade700 -1 osd.3 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:00.584422+0000 front 2024-05-08T06:38:00.584327+0000 (oldest deadline 2024-05-08T06:38:22.884398+0000) 2024-05-08T06:38:24.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 5.25 MB/sec execute 45 sec latency 28457.921 ms 2024-05-08T06:38:24.917 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:38:24 smithi082 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-3[43962]: 2024-05-08T06:38:24.790+0000 7fda9bade700 -1 osd.3 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:00.584422+0000 front 2024-05-08T06:38:00.584327+0000 (oldest deadline 2024-05-08T06:38:22.884398+0000) 2024-05-08T06:38:25.278 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:24 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:24.788+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:37:57.621161+0000 front 2024-05-08T06:37:57.616129+0000 (oldest deadline 2024-05-08T06:38:24.016676+0000) 2024-05-08T06:38:25.433 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 5.13 MB/sec execute 46 sec latency 29457.973 ms 2024-05-08T06:38:26.028 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:25 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:25.744+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:37:57.621161+0000 front 2024-05-08T06:37:57.616129+0000 (oldest deadline 2024-05-08T06:38:24.016676+0000) 2024-05-08T06:38:26.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:25 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:25.946+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:26.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 5.02 MB/sec execute 47 sec latency 30457.995 ms 2024-05-08T06:38:26.514 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:38:26 smithi082 podman[56576]: 2024-05-08 06:38:26.244953751 +0000 UTC m=+5.597900065 container died 7f3b8f015f5975e9cd043b819ead0d67251f4e887dd5d27a3038a3b4e4bad4af (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-3, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.k8s.display-name=CentOS Stream 8, name=centos-stream, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, RELEASE=HEAD, GIT_BRANCH=HEAD, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.buildah.version=1.19.8) 2024-05-08T06:38:26.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:26 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:26.204+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:26.765 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:38:26 smithi082 podman[56576]: 2024-05-08 06:38:26.590797357 +0000 UTC m=+5.943743664 container cleanup 7f3b8f015f5975e9cd043b819ead0d67251f4e887dd5d27a3038a3b4e4bad4af (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-3, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, version=8, com.redhat.component=centos-stream-container, distribution-scope=public, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_CLEAN=True, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, release=754, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, architecture=x86_64) 2024-05-08T06:38:26.765 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:38:26 smithi082 bash[56576]: ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-3 2024-05-08T06:38:27.028 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:26 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:26.776+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:37:57.621161+0000 front 2024-05-08T06:37:57.616129+0000 (oldest deadline 2024-05-08T06:38:24.016676+0000) 2024-05-08T06:38:27.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:26 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:26.951+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:27.053 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:38:26 smithi082 podman[56596]: 2024-05-08 06:38:26.767026016 +0000 UTC m=+0.514875987 container remove 7f3b8f015f5975e9cd043b819ead0d67251f4e887dd5d27a3038a3b4e4bad4af (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-3, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, ceph=True, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, version=8, architecture=x86_64, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vcs-type=git, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream) 2024-05-08T06:38:27.053 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:38:26 smithi082 bash[56608]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd.3" found: no such container 2024-05-08T06:38:27.343 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:38:27 smithi082 podman[56672]: 2024-05-08 06:38:26.958432471 +0000 UTC m=+0.018647024 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-08T06:38:27.343 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:38:27 smithi082 podman[56672]: 2024-05-08 06:38:27.059470879 +0000 UTC m=+0.119685429 container create b9002ceb88415929bbe4c7058909a47aad01c77110df5262b1dc54f2d6af3003 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-3-deactivate, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, release=754, build-date=2022-05-03T08:36:31.336870, ceph=True, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, maintainer=Guillaume Abrioux , io.openshift.expose-services=, com.redhat.component=centos-stream-container, name=centos-stream, io.openshift.tags=base centos centos-stream, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-05-08T06:38:27.344 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:38:27 smithi082 podman[56672]: 2024-05-08 06:38:27.261462423 +0000 UTC m=+0.321676967 container init b9002ceb88415929bbe4c7058909a47aad01c77110df5262b1dc54f2d6af3003 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-3-deactivate, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, release=754, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, ceph=True, com.redhat.component=centos-stream-container, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc.) 2024-05-08T06:38:27.344 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:38:27 smithi082 podman[56672]: 2024-05-08 06:38:27.264975318 +0000 UTC m=+0.325189862 container start b9002ceb88415929bbe4c7058909a47aad01c77110df5262b1dc54f2d6af3003 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-3-deactivate, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, RELEASE=HEAD, GIT_BRANCH=HEAD, name=centos-stream, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, architecture=x86_64, maintainer=Guillaume Abrioux , io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container) 2024-05-08T06:38:27.344 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:38:27 smithi082 podman[56672]: 2024-05-08 06:38:27.345152828 +0000 UTC m=+0.405367380 container attach b9002ceb88415929bbe4c7058909a47aad01c77110df5262b1dc54f2d6af3003 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-3-deactivate, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, vcs-type=git, RELEASE=HEAD, distribution-scope=public, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream) 2024-05-08T06:38:27.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 4.92 MB/sec execute 48 sec latency 31458.266 ms 2024-05-08T06:38:27.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:27 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:27.204+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:27.917 INFO:journalctl@ceph.osd.3.smithi082.stdout:May 08 06:38:27 smithi082 podman[56672]: 2024-05-08 06:38:27.615515901 +0000 UTC m=+0.675730453 container died b9002ceb88415929bbe4c7058909a47aad01c77110df5262b1dc54f2d6af3003 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-3-deactivate, version=8, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.component=centos-stream-container, name=centos-stream, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, vcs-type=git) 2024-05-08T06:38:28.154 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:27 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:27.796+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:37:57.621161+0000 front 2024-05-08T06:37:57.616129+0000 (oldest deadline 2024-05-08T06:38:24.016676+0000) 2024-05-08T06:38:28.155 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:27 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:27.938+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:28.284 DEBUG:teuthology.orchestra.run.smithi082:> sudo pkill -f 'journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.3.service' 2024-05-08T06:38:28.321 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-08T06:38:28.322 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-05-08T06:38:28.322 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-05-08T06:38:28.322 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.4 2024-05-08T06:38:28.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 4.82 MB/sec execute 49 sec latency 32458.292 ms 2024-05-08T06:38:28.522 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:28 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:28.215+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:28.522 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:28 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4[29252]: 2024-05-08T06:38:28.155+0000 7efd7f74a700 -1 osd.4 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:02.113997+0000 front 2024-05-08T06:38:02.114014+0000 (oldest deadline 2024-05-08T06:38:28.014166+0000) 2024-05-08T06:38:28.523 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:28 smithi134 systemd[1]: Stopping Ceph osd.4 for 02181fb0-0d04-11ef-bc97-c7b262605968... 2024-05-08T06:38:28.523 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:28 smithi134 bash[41278]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd.4" found: no such container 2024-05-08T06:38:28.523 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:28 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4[29252]: 2024-05-08T06:38:28.523+0000 7efd855b4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-08T06:38:28.523 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:28 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4[29252]: 2024-05-08T06:38:28.523+0000 7efd855b4700 -1 osd.4 70 *** Got signal Terminated *** 2024-05-08T06:38:28.523 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:28 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4[29252]: 2024-05-08T06:38:28.523+0000 7efd855b4700 -1 osd.4 70 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-08T06:38:28.778 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:28 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:28.758+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:37:57.621161+0000 front 2024-05-08T06:37:57.616129+0000 (oldest deadline 2024-05-08T06:38:24.016676+0000) 2024-05-08T06:38:29.203 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:28 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:28.946+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:29.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 4.72 MB/sec execute 50 sec latency 33458.326 ms 2024-05-08T06:38:29.461 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:29 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4[29252]: 2024-05-08T06:38:29.204+0000 7efd7f74a700 -1 osd.4 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:02.113997+0000 front 2024-05-08T06:38:02.114014+0000 (oldest deadline 2024-05-08T06:38:28.014166+0000) 2024-05-08T06:38:29.461 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:29 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:29.220+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:30.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:29 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:29.927+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:30.028 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:29 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:29.770+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:37:57.621161+0000 front 2024-05-08T06:37:57.616129+0000 (oldest deadline 2024-05-08T06:38:24.016676+0000) 2024-05-08T06:38:30.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 4.63 MB/sec execute 51 sec latency 34458.376 ms 2024-05-08T06:38:30.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:30 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:30.176+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:30.528 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:30 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4[29252]: 2024-05-08T06:38:30.211+0000 7efd7f74a700 -1 osd.4 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:02.113997+0000 front 2024-05-08T06:38:02.114014+0000 (oldest deadline 2024-05-08T06:38:28.014166+0000) 2024-05-08T06:38:31.028 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:30 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:30.721+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:37:57.621161+0000 front 2024-05-08T06:37:57.616129+0000 (oldest deadline 2024-05-08T06:38:24.016676+0000) 2024-05-08T06:38:31.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:30 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:30.951+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:31.029 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:30 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:30.951+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:31.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 4.54 MB/sec execute 52 sec latency 35458.410 ms 2024-05-08T06:38:31.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:31 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:31.182+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:31.528 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:31 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4[29252]: 2024-05-08T06:38:31.206+0000 7efd7f74a700 -1 osd.4 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:02.113997+0000 front 2024-05-08T06:38:02.114014+0000 (oldest deadline 2024-05-08T06:38:28.014166+0000) 2024-05-08T06:38:31.528 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:31 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4[29252]: 2024-05-08T06:38:31.206+0000 7efd7f74a700 -1 osd.4 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:08.014547+0000 front 2024-05-08T06:38:08.014570+0000 (oldest deadline 2024-05-08T06:38:30.914788+0000) 2024-05-08T06:38:32.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:31 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:31.912+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:32.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:31 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:31.912+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:32.028 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:31 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:31.727+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:37:57.621161+0000 front 2024-05-08T06:37:57.616129+0000 (oldest deadline 2024-05-08T06:38:24.016676+0000) 2024-05-08T06:38:32.028 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:31 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:31.727+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:07.925279+0000 front 2024-05-08T06:38:07.925217+0000 (oldest deadline 2024-05-08T06:38:31.418681+0000) 2024-05-08T06:38:32.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 4.46 MB/sec execute 53 sec latency 36458.449 ms 2024-05-08T06:38:32.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:32 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:32.160+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:32.528 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:32 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4[29252]: 2024-05-08T06:38:32.230+0000 7efd7f74a700 -1 osd.4 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:02.113997+0000 front 2024-05-08T06:38:02.114014+0000 (oldest deadline 2024-05-08T06:38:28.014166+0000) 2024-05-08T06:38:32.528 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:32 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4[29252]: 2024-05-08T06:38:32.230+0000 7efd7f74a700 -1 osd.4 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:08.014547+0000 front 2024-05-08T06:38:08.014570+0000 (oldest deadline 2024-05-08T06:38:30.914788+0000) 2024-05-08T06:38:33.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:32 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:32.901+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:33.029 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:32 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:32.901+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:33.029 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:32 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:32.766+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:37:57.621161+0000 front 2024-05-08T06:37:57.616129+0000 (oldest deadline 2024-05-08T06:38:24.016676+0000) 2024-05-08T06:38:33.029 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:32 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:32.766+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:07.925279+0000 front 2024-05-08T06:38:07.925217+0000 (oldest deadline 2024-05-08T06:38:31.418681+0000) 2024-05-08T06:38:33.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 4.37 MB/sec execute 54 sec latency 37458.495 ms 2024-05-08T06:38:33.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:33 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:33.111+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:33.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:33 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:33.111+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:06.440504+0000 front 2024-05-08T06:38:06.446638+0000 (oldest deadline 2024-05-08T06:38:32.340532+0000) 2024-05-08T06:38:33.528 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:33 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4[29252]: 2024-05-08T06:38:33.222+0000 7efd7f74a700 -1 osd.4 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:02.113997+0000 front 2024-05-08T06:38:02.114014+0000 (oldest deadline 2024-05-08T06:38:28.014166+0000) 2024-05-08T06:38:33.528 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:33 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4[29252]: 2024-05-08T06:38:33.222+0000 7efd7f74a700 -1 osd.4 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:08.014547+0000 front 2024-05-08T06:38:08.014570+0000 (oldest deadline 2024-05-08T06:38:30.914788+0000) 2024-05-08T06:38:34.124 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:33 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:33.951+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:34.125 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:33 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:33.951+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:34.125 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:33 smithi134 podman[41291]: 2024-05-08 06:38:33.912953666 +0000 UTC m=+5.505070881 container died 3de640027e4e5fe81b7fa766c1e0212a0bced63874af1ac42f9d6147e166c0d9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, vcs-type=git, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, architecture=x86_64, ceph=True, io.openshift.tags=base centos centos-stream, release=754, io.buildah.version=1.19.8) 2024-05-08T06:38:34.125 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:33 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:33.789+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:37:57.621161+0000 front 2024-05-08T06:37:57.616129+0000 (oldest deadline 2024-05-08T06:38:24.016676+0000) 2024-05-08T06:38:34.125 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:33 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:33.789+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:07.925279+0000 front 2024-05-08T06:38:07.925217+0000 (oldest deadline 2024-05-08T06:38:31.418681+0000) 2024-05-08T06:38:34.392 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:34 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:34.125+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:34.392 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:34 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:34.125+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:06.440504+0000 front 2024-05-08T06:38:06.446638+0000 (oldest deadline 2024-05-08T06:38:32.340532+0000) 2024-05-08T06:38:34.392 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:34 smithi134 podman[41291]: 2024-05-08 06:38:34.364775907 +0000 UTC m=+5.956893167 container cleanup 3de640027e4e5fe81b7fa766c1e0212a0bced63874af1ac42f9d6147e166c0d9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4, name=centos-stream, release=754, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.openshift.tags=base centos centos-stream, distribution-scope=public, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.buildah.version=1.19.8, RELEASE=HEAD) 2024-05-08T06:38:34.393 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:34 smithi134 bash[41291]: ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4 2024-05-08T06:38:34.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 4.29 MB/sec execute 55 sec latency 38458.546 ms 2024-05-08T06:38:34.775 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:34 smithi134 bash[41334]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd.4" found: no such container 2024-05-08T06:38:34.776 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:34 smithi134 podman[41321]: 2024-05-08 06:38:34.641252785 +0000 UTC m=+0.723350669 container remove 3de640027e4e5fe81b7fa766c1e0212a0bced63874af1ac42f9d6147e166c0d9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., version=8, com.redhat.component=centos-stream-container, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, ceph=True, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD) 2024-05-08T06:38:35.028 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:34 smithi134 podman[41398]: 2024-05-08 06:38:34.715370243 +0000 UTC m=+0.019077670 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-08T06:38:35.028 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:34 smithi134 podman[41398]: 2024-05-08 06:38:34.825497833 +0000 UTC m=+0.129205261 container create 31f0c4c824552879da727219117aa63a73de0a4c8d781470384a52f811da59d9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, io.openshift.expose-services=, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, vcs-type=git, io.buildah.version=1.19.8, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, ceph=True, GIT_BRANCH=HEAD, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-05-08T06:38:35.029 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:34 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:34.776+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:37:57.621161+0000 front 2024-05-08T06:37:57.616129+0000 (oldest deadline 2024-05-08T06:38:24.016676+0000) 2024-05-08T06:38:35.029 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:34 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:34.776+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:07.925279+0000 front 2024-05-08T06:38:07.925217+0000 (oldest deadline 2024-05-08T06:38:31.418681+0000) 2024-05-08T06:38:35.029 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:34 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:34.907+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:35.029 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:34 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:34.907+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:35.293 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:35 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:35.167+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:35.293 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:35 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:35.167+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:06.440504+0000 front 2024-05-08T06:38:06.446638+0000 (oldest deadline 2024-05-08T06:38:32.340532+0000) 2024-05-08T06:38:35.293 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:35 smithi134 podman[41398]: 2024-05-08 06:38:35.053187009 +0000 UTC m=+0.356894444 container init 31f0c4c824552879da727219117aa63a73de0a4c8d781470384a52f811da59d9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, architecture=x86_64, GIT_CLEAN=True, io.buildah.version=1.19.8, ceph=True, RELEASE=HEAD, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, com.redhat.component=centos-stream-container, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git) 2024-05-08T06:38:35.294 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:35 smithi134 podman[41398]: 2024-05-08 06:38:35.056607985 +0000 UTC m=+0.360315414 container start 31f0c4c824552879da727219117aa63a73de0a4c8d781470384a52f811da59d9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4-deactivate, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, version=8, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-05-08T06:38:35.294 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:35 smithi134 podman[41398]: 2024-05-08 06:38:35.294922503 +0000 UTC m=+0.598629957 container attach 31f0c4c824552879da727219117aa63a73de0a4c8d781470384a52f811da59d9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4-deactivate, architecture=x86_64, vcs-type=git, maintainer=Guillaume Abrioux , RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, version=8, io.buildah.version=1.19.8, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_CLEAN=True, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream) 2024-05-08T06:38:35.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 4.22 MB/sec execute 56 sec latency 39458.598 ms 2024-05-08T06:38:35.697 INFO:journalctl@ceph.osd.4.smithi134.stdout:May 08 06:38:35 smithi134 podman[41398]: 2024-05-08 06:38:35.421859861 +0000 UTC m=+0.725567293 container died 31f0c4c824552879da727219117aa63a73de0a4c8d781470384a52f811da59d9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, architecture=x86_64, io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-05-08T06:38:36.028 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:35 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:35.805+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:37:57.621161+0000 front 2024-05-08T06:37:57.616129+0000 (oldest deadline 2024-05-08T06:38:24.016676+0000) 2024-05-08T06:38:36.028 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:35 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:35.805+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:07.925279+0000 front 2024-05-08T06:38:07.925217+0000 (oldest deadline 2024-05-08T06:38:31.418681+0000) 2024-05-08T06:38:36.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:35 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:35.932+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:36.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:35 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:35.932+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:36.082 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.4.service' 2024-05-08T06:38:36.126 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-08T06:38:36.127 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-05-08T06:38:36.127 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-05-08T06:38:36.127 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.5 2024-05-08T06:38:36.350 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:36 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:36.199+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:36.350 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:36 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:36.199+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:06.440504+0000 front 2024-05-08T06:38:06.446638+0000 (oldest deadline 2024-05-08T06:38:32.340532+0000) 2024-05-08T06:38:36.351 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:36 smithi134 systemd[1]: Stopping Ceph osd.5 for 02181fb0-0d04-11ef-bc97-c7b262605968... 2024-05-08T06:38:36.351 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:36 smithi134 bash[41477]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd.5" found: no such container 2024-05-08T06:38:36.351 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:36 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:36.351+0000 7f803c49e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-08T06:38:36.351 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:36 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:36.351+0000 7f803c49e700 -1 osd.5 70 *** Got signal Terminated *** 2024-05-08T06:38:36.351 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:36 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:36.351+0000 7f803c49e700 -1 osd.5 70 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-08T06:38:36.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 4.14 MB/sec execute 57 sec latency 40458.659 ms 2024-05-08T06:38:37.028 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:36 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:36.767+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:37:57.621161+0000 front 2024-05-08T06:37:57.616129+0000 (oldest deadline 2024-05-08T06:38:24.016676+0000) 2024-05-08T06:38:37.028 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:36 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:36.767+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:07.925279+0000 front 2024-05-08T06:38:07.925217+0000 (oldest deadline 2024-05-08T06:38:31.418681+0000) 2024-05-08T06:38:37.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:36 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:36.906+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:37.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:36 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:36.906+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:37.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 4.07 MB/sec execute 58 sec latency 41458.705 ms 2024-05-08T06:38:37.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:37 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:37.170+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:37.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:37 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:37.170+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:06.440504+0000 front 2024-05-08T06:38:06.446638+0000 (oldest deadline 2024-05-08T06:38:32.340532+0000) 2024-05-08T06:38:38.028 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:37 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:37.767+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:37:57.621161+0000 front 2024-05-08T06:37:57.616129+0000 (oldest deadline 2024-05-08T06:38:24.016676+0000) 2024-05-08T06:38:38.028 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:37 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:37.767+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:07.925279+0000 front 2024-05-08T06:38:07.925217+0000 (oldest deadline 2024-05-08T06:38:31.418681+0000) 2024-05-08T06:38:38.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:37 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:37.905+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:38.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:37 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:37.905+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:38.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 4.00 MB/sec execute 59 sec latency 42458.758 ms 2024-05-08T06:38:38.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:38 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:38.205+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:38.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:38 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:38.205+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:06.440504+0000 front 2024-05-08T06:38:06.446638+0000 (oldest deadline 2024-05-08T06:38:32.340532+0000) 2024-05-08T06:38:39.028 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:38 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:38.745+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:37:57.621161+0000 front 2024-05-08T06:37:57.616129+0000 (oldest deadline 2024-05-08T06:38:24.016676+0000) 2024-05-08T06:38:39.028 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:38 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:38.745+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:07.925279+0000 front 2024-05-08T06:38:07.925217+0000 (oldest deadline 2024-05-08T06:38:31.418681+0000) 2024-05-08T06:38:39.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:38 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:38.947+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:39.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:38 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:38.947+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:39.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 3.94 MB/sec execute 60 sec latency 43458.805 ms 2024-05-08T06:38:39.472 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:39 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:39.217+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:39.472 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:39 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:39.217+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:06.440504+0000 front 2024-05-08T06:38:06.446638+0000 (oldest deadline 2024-05-08T06:38:32.340532+0000) 2024-05-08T06:38:39.472 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:39 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:39.217+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.842401+0000 front 2024-05-08T06:38:16.843363+0000 (oldest deadline 2024-05-08T06:38:38.542517+0000) 2024-05-08T06:38:40.028 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:39 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:39.741+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:37:57.621161+0000 front 2024-05-08T06:37:57.616129+0000 (oldest deadline 2024-05-08T06:38:24.016676+0000) 2024-05-08T06:38:40.028 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:39 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:39.741+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:07.925279+0000 front 2024-05-08T06:38:07.925217+0000 (oldest deadline 2024-05-08T06:38:31.418681+0000) 2024-05-08T06:38:40.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:39 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:39.903+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:40.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:39 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:39.903+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:40.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 3.87 MB/sec execute 61 sec latency 44458.826 ms 2024-05-08T06:38:40.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:40 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:40.211+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:40.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:40 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:40.211+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:06.440504+0000 front 2024-05-08T06:38:06.446638+0000 (oldest deadline 2024-05-08T06:38:32.340532+0000) 2024-05-08T06:38:40.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:40 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:40.211+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.842401+0000 front 2024-05-08T06:38:16.843363+0000 (oldest deadline 2024-05-08T06:38:38.542517+0000) 2024-05-08T06:38:41.028 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:40 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:40.776+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:37:57.621161+0000 front 2024-05-08T06:37:57.616129+0000 (oldest deadline 2024-05-08T06:38:24.016676+0000) 2024-05-08T06:38:41.028 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:40 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:40.776+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:07.925279+0000 front 2024-05-08T06:38:07.925217+0000 (oldest deadline 2024-05-08T06:38:31.418681+0000) 2024-05-08T06:38:41.028 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:40 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5[32300]: 2024-05-08T06:38:40.776+0000 7f803963a700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:17.125114+0000 front 2024-05-08T06:38:17.121080+0000 (oldest deadline 2024-05-08T06:38:40.021198+0000) 2024-05-08T06:38:41.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:40 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:40.902+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:41.029 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:40 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:40.902+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:41.029 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:40 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:40.902+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.131386+0000 front 2024-05-08T06:38:16.125303+0000 (oldest deadline 2024-05-08T06:38:40.825440+0000) 2024-05-08T06:38:41.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 3.81 MB/sec execute 62 sec latency 45458.867 ms 2024-05-08T06:38:41.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:41 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:41.245+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:41.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:41 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:41.245+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:06.440504+0000 front 2024-05-08T06:38:06.446638+0000 (oldest deadline 2024-05-08T06:38:32.340532+0000) 2024-05-08T06:38:41.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:41 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:41.245+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.842401+0000 front 2024-05-08T06:38:16.843363+0000 (oldest deadline 2024-05-08T06:38:38.542517+0000) 2024-05-08T06:38:42.217 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:41 smithi134 podman[41489]: 2024-05-08 06:38:41.940699247 +0000 UTC m=+5.689267320 container died 75f363c21b07a8a2bece5264d8445db967a203cf93b2673317c323c0de2c0c66 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, com.redhat.component=centos-stream-container, distribution-scope=public, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, release=754, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870) 2024-05-08T06:38:42.217 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:41 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:41.853+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:42.217 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:41 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:41.853+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:42.217 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:41 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:41.853+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.131386+0000 front 2024-05-08T06:38:16.125303+0000 (oldest deadline 2024-05-08T06:38:40.825440+0000) 2024-05-08T06:38:42.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 3.75 MB/sec execute 63 sec latency 46458.911 ms 2024-05-08T06:38:42.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:42 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:42.264+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:42.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:42 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:42.264+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:06.440504+0000 front 2024-05-08T06:38:06.446638+0000 (oldest deadline 2024-05-08T06:38:32.340532+0000) 2024-05-08T06:38:42.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:42 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:42.264+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.842401+0000 front 2024-05-08T06:38:16.843363+0000 (oldest deadline 2024-05-08T06:38:38.542517+0000) 2024-05-08T06:38:42.529 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:42 smithi134 podman[41489]: 2024-05-08 06:38:42.426228019 +0000 UTC m=+6.174796086 container cleanup 75f363c21b07a8a2bece5264d8445db967a203cf93b2673317c323c0de2c0c66 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, GIT_CLEAN=True, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, vcs-type=git, io.buildah.version=1.19.8, distribution-scope=public, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, version=8, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-05-08T06:38:42.529 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:42 smithi134 bash[41489]: ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5 2024-05-08T06:38:42.529 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:42 smithi134 bash[41519]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd.5" found: no such container 2024-05-08T06:38:42.981 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:42 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:42.814+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:42.981 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:42 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:42.814+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:42.982 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:42 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:42.814+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.131386+0000 front 2024-05-08T06:38:16.125303+0000 (oldest deadline 2024-05-08T06:38:40.825440+0000) 2024-05-08T06:38:42.982 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:42 smithi134 podman[41504]: 2024-05-08 06:38:42.660899162 +0000 UTC m=+0.715472929 container remove 75f363c21b07a8a2bece5264d8445db967a203cf93b2673317c323c0de2c0c66 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, distribution-scope=public, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, name=centos-stream, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, version=8, architecture=x86_64, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870) 2024-05-08T06:38:42.982 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:42 smithi134 podman[41580]: 2024-05-08 06:38:42.765059388 +0000 UTC m=+0.018961070 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-08T06:38:42.982 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:42 smithi134 podman[41580]: 2024-05-08 06:38:42.870174403 +0000 UTC m=+0.124076119 container create 1f42c3a31764ceec622ab0aa0ae5e16609a952f87657e4476d3ed98d2e5cf1d6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5-deactivate, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, com.redhat.component=centos-stream-container, ceph=True, distribution-scope=public, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., version=8, release=754, name=centos-stream, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream) 2024-05-08T06:38:43.278 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:43 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:43.230+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:43.278 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:43 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:43.230+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:06.440504+0000 front 2024-05-08T06:38:06.446638+0000 (oldest deadline 2024-05-08T06:38:32.340532+0000) 2024-05-08T06:38:43.278 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:43 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:43.230+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.842401+0000 front 2024-05-08T06:38:16.843363+0000 (oldest deadline 2024-05-08T06:38:38.542517+0000) 2024-05-08T06:38:43.278 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:43 smithi134 podman[41580]: 2024-05-08 06:38:43.072385824 +0000 UTC m=+0.326287494 container init 1f42c3a31764ceec622ab0aa0ae5e16609a952f87657e4476d3ed98d2e5cf1d6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5-deactivate, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, RELEASE=HEAD, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_BRANCH=HEAD, distribution-scope=public, io.openshift.expose-services=, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, io.openshift.tags=base centos centos-stream, version=8, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870) 2024-05-08T06:38:43.279 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:43 smithi134 podman[41580]: 2024-05-08 06:38:43.075775164 +0000 UTC m=+0.329676838 container start 1f42c3a31764ceec622ab0aa0ae5e16609a952f87657e4476d3ed98d2e5cf1d6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5-deactivate, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, maintainer=Guillaume Abrioux , version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, architecture=x86_64, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True) 2024-05-08T06:38:43.279 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:43 smithi134 podman[41580]: 2024-05-08 06:38:43.147153824 +0000 UTC m=+0.401055508 container attach 1f42c3a31764ceec622ab0aa0ae5e16609a952f87657e4476d3ed98d2e5cf1d6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5-deactivate, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, vcs-type=git, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , version=8, name=centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, ceph=True) 2024-05-08T06:38:43.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 3.69 MB/sec execute 64 sec latency 47458.939 ms 2024-05-08T06:38:43.648 INFO:journalctl@ceph.osd.5.smithi134.stdout:May 08 06:38:43 smithi134 podman[41580]: 2024-05-08 06:38:43.368060383 +0000 UTC m=+0.621962067 container died 1f42c3a31764ceec622ab0aa0ae5e16609a952f87657e4476d3ed98d2e5cf1d6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-5-deactivate, architecture=x86_64, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, maintainer=Guillaume Abrioux , vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754) 2024-05-08T06:38:44.016 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:43 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:43.808+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:44.016 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:43 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:43.808+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:44.016 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:43 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:43.808+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.131386+0000 front 2024-05-08T06:38:16.125303+0000 (oldest deadline 2024-05-08T06:38:40.825440+0000) 2024-05-08T06:38:44.039 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.5.service' 2024-05-08T06:38:44.078 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-08T06:38:44.078 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-05-08T06:38:44.078 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-05-08T06:38:44.079 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.6 2024-05-08T06:38:44.278 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:44 smithi134 systemd[1]: Stopping Ceph osd.6 for 02181fb0-0d04-11ef-bc97-c7b262605968... 2024-05-08T06:38:44.278 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:44 smithi134 bash[41659]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd.6" found: no such container 2024-05-08T06:38:44.278 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:44 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:44.221+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:44.278 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:44 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:44.221+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:06.440504+0000 front 2024-05-08T06:38:06.446638+0000 (oldest deadline 2024-05-08T06:38:32.340532+0000) 2024-05-08T06:38:44.279 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:44 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:44.221+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.842401+0000 front 2024-05-08T06:38:16.843363+0000 (oldest deadline 2024-05-08T06:38:38.542517+0000) 2024-05-08T06:38:44.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 3.63 MB/sec execute 65 sec latency 48458.988 ms 2024-05-08T06:38:44.778 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:44 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:44.318+0000 7f0aeaa83700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-08T06:38:44.778 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:44 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:44.318+0000 7f0aeaa83700 -1 osd.6 70 *** Got signal Terminated *** 2024-05-08T06:38:44.779 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:44 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:44.318+0000 7f0aeaa83700 -1 osd.6 70 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-08T06:38:45.261 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:44 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:44.829+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:45.261 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:44 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:44.829+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:45.261 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:44 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:44.829+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.131386+0000 front 2024-05-08T06:38:16.125303+0000 (oldest deadline 2024-05-08T06:38:40.825440+0000) 2024-05-08T06:38:45.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 3.58 MB/sec execute 66 sec latency 49459.012 ms 2024-05-08T06:38:45.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:45 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:45.261+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:45.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:45 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:45.261+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:06.440504+0000 front 2024-05-08T06:38:06.446638+0000 (oldest deadline 2024-05-08T06:38:32.340532+0000) 2024-05-08T06:38:45.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:45 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:45.261+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.842401+0000 front 2024-05-08T06:38:16.843363+0000 (oldest deadline 2024-05-08T06:38:38.542517+0000) 2024-05-08T06:38:46.224 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:45 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:45.814+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:46.225 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:45 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:45.814+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:46.225 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:45 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:45.814+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.131386+0000 front 2024-05-08T06:38:16.125303+0000 (oldest deadline 2024-05-08T06:38:40.825440+0000) 2024-05-08T06:38:46.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 3.52 MB/sec execute 67 sec latency 50459.080 ms 2024-05-08T06:38:46.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:46 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:46.225+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:46.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:46 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:46.225+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:06.440504+0000 front 2024-05-08T06:38:06.446638+0000 (oldest deadline 2024-05-08T06:38:32.340532+0000) 2024-05-08T06:38:46.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:46 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:46.225+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.842401+0000 front 2024-05-08T06:38:16.843363+0000 (oldest deadline 2024-05-08T06:38:38.542517+0000) 2024-05-08T06:38:47.220 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:46 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:46.792+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:47.220 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:46 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:46.792+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:47.220 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:46 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:46.792+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.131386+0000 front 2024-05-08T06:38:16.125303+0000 (oldest deadline 2024-05-08T06:38:40.825440+0000) 2024-05-08T06:38:47.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 3.47 MB/sec execute 68 sec latency 51459.098 ms 2024-05-08T06:38:47.527 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:47 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:47.221+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:47.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:47 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:47.221+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:06.440504+0000 front 2024-05-08T06:38:06.446638+0000 (oldest deadline 2024-05-08T06:38:32.340532+0000) 2024-05-08T06:38:47.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:47 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:47.221+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.842401+0000 front 2024-05-08T06:38:16.843363+0000 (oldest deadline 2024-05-08T06:38:38.542517+0000) 2024-05-08T06:38:48.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:47 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:47.748+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:48.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:47 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:47.748+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:48.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:47 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:47.748+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.131386+0000 front 2024-05-08T06:38:16.125303+0000 (oldest deadline 2024-05-08T06:38:40.825440+0000) 2024-05-08T06:38:48.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 3.42 MB/sec execute 69 sec latency 52459.120 ms 2024-05-08T06:38:48.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:48 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:48.196+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:48.528 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:48 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:48.196+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:06.440504+0000 front 2024-05-08T06:38:06.446638+0000 (oldest deadline 2024-05-08T06:38:32.340532+0000) 2024-05-08T06:38:48.529 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:48 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:48.196+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.842401+0000 front 2024-05-08T06:38:16.843363+0000 (oldest deadline 2024-05-08T06:38:38.542517+0000) 2024-05-08T06:38:48.529 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:48 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:48.196+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2024-05-08T06:38:24.943874+0000 front 2024-05-08T06:38:24.947715+0000 (oldest deadline 2024-05-08T06:38:47.843963+0000) 2024-05-08T06:38:49.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:48 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:48.718+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:49.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:48 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:48.718+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:49.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:48 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:48.718+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.131386+0000 front 2024-05-08T06:38:16.125303+0000 (oldest deadline 2024-05-08T06:38:40.825440+0000) 2024-05-08T06:38:49.434 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 3.37 MB/sec execute 70 sec latency 53459.145 ms 2024-05-08T06:38:49.471 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:49 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:49.218+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.243591+0000 front 2024-05-08T06:38:01.243511+0000 (oldest deadline 2024-05-08T06:38:25.939607+0000) 2024-05-08T06:38:49.472 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:49 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:49.218+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:06.440504+0000 front 2024-05-08T06:38:06.446638+0000 (oldest deadline 2024-05-08T06:38:32.340532+0000) 2024-05-08T06:38:49.472 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:49 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:49.218+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.842401+0000 front 2024-05-08T06:38:16.843363+0000 (oldest deadline 2024-05-08T06:38:38.542517+0000) 2024-05-08T06:38:49.472 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:49 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6[35275]: 2024-05-08T06:38:49.218+0000 7f0ae4c19700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2024-05-08T06:38:24.943874+0000 front 2024-05-08T06:38:24.947715+0000 (oldest deadline 2024-05-08T06:38:47.843963+0000) 2024-05-08T06:38:49.723 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:49 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:49.719+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:49.724 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:49 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:49.719+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:49.724 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:49 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:49.719+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.131386+0000 front 2024-05-08T06:38:16.125303+0000 (oldest deadline 2024-05-08T06:38:40.825440+0000) 2024-05-08T06:38:49.724 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:49 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:49.720+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2024-05-08T06:38:21.926513+0000 front 2024-05-08T06:38:21.926485+0000 (oldest deadline 2024-05-08T06:38:48.927152+0000) 2024-05-08T06:38:50.002 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:49 smithi134 podman[41670]: 2024-05-08 06:38:49.725353673 +0000 UTC m=+5.536144450 container died 98bcf9455cd66f0a31fb44c4d32b417fdbc926c9b196106c10f3e0af5fb20d9e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, version=8, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_CLEAN=True, vcs-type=git, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD) 2024-05-08T06:38:50.278 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:50 smithi134 podman[41670]: 2024-05-08 06:38:50.153606854 +0000 UTC m=+5.964397670 container cleanup 98bcf9455cd66f0a31fb44c4d32b417fdbc926c9b196106c10f3e0af5fb20d9e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , ceph=True, distribution-scope=public, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, architecture=x86_64, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-05-08T06:38:50.278 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:50 smithi134 bash[41670]: ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6 2024-05-08T06:38:50.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 3.33 MB/sec execute 71 sec latency 54459.164 ms 2024-05-08T06:38:50.594 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:50 smithi134 podman[41687]: 2024-05-08 06:38:50.338061005 +0000 UTC m=+0.607430332 container remove 98bcf9455cd66f0a31fb44c4d32b417fdbc926c9b196106c10f3e0af5fb20d9e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.buildah.version=1.19.8, vcs-type=git, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, GIT_BRANCH=HEAD, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0) 2024-05-08T06:38:50.594 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:50 smithi134 bash[41700]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd.6" found: no such container 2024-05-08T06:38:50.847 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:50 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:50.676+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:50.848 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:50 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:50.676+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:50.848 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:50 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:50.676+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.131386+0000 front 2024-05-08T06:38:16.125303+0000 (oldest deadline 2024-05-08T06:38:40.825440+0000) 2024-05-08T06:38:50.848 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:50 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:50.676+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2024-05-08T06:38:21.926513+0000 front 2024-05-08T06:38:21.926485+0000 (oldest deadline 2024-05-08T06:38:48.927152+0000) 2024-05-08T06:38:50.848 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:50 smithi134 podman[41764]: 2024-05-08 06:38:50.500123574 +0000 UTC m=+0.019382696 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-08T06:38:50.848 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:50 smithi134 podman[41764]: 2024-05-08 06:38:50.605531853 +0000 UTC m=+0.124790955 container create 80fcf3388a7853f346d7158fb2930b0fe84252bf5d29f99ba49dfdc31d61edb4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6-deactivate, name=centos-stream, release=754, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=HEAD, version=8) 2024-05-08T06:38:50.848 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:50 smithi134 podman[41764]: 2024-05-08 06:38:50.849218019 +0000 UTC m=+0.368477136 container init 80fcf3388a7853f346d7158fb2930b0fe84252bf5d29f99ba49dfdc31d61edb4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6-deactivate, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, ceph=True, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, version=8, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0) 2024-05-08T06:38:51.140 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:50 smithi134 podman[41764]: 2024-05-08 06:38:50.852889374 +0000 UTC m=+0.372148477 container start 80fcf3388a7853f346d7158fb2930b0fe84252bf5d29f99ba49dfdc31d61edb4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6-deactivate, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, vcs-type=git, maintainer=Guillaume Abrioux , io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, release=754, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-08T06:38:51.140 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:50 smithi134 podman[41764]: 2024-05-08 06:38:50.940983099 +0000 UTC m=+0.460242215 container attach 80fcf3388a7853f346d7158fb2930b0fe84252bf5d29f99ba49dfdc31d61edb4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6-deactivate, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, io.openshift.expose-services=, name=centos-stream, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, version=8, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, release=754, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, ceph=True) 2024-05-08T06:38:51.434 INFO:journalctl@ceph.osd.6.smithi134.stdout:May 08 06:38:51 smithi134 podman[41764]: 2024-05-08 06:38:51.14182413 +0000 UTC m=+0.661083236 container died 80fcf3388a7853f346d7158fb2930b0fe84252bf5d29f99ba49dfdc31d61edb4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-6-deactivate, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, vcs-type=git, io.openshift.expose-services=, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, ceph=True, release=754, GIT_CLEAN=True, maintainer=Guillaume Abrioux , name=centos-stream, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD) 2024-05-08T06:38:51.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 3.28 MB/sec execute 72 sec latency 55459.226 ms 2024-05-08T06:38:51.720 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.6.service' 2024-05-08T06:38:51.758 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-08T06:38:51.758 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-05-08T06:38:51.758 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-05-08T06:38:51.758 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.7 2024-05-08T06:38:51.979 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:51 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:51.712+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:51.980 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:51 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:51.712+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:51.980 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:51 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:51.712+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.131386+0000 front 2024-05-08T06:38:16.125303+0000 (oldest deadline 2024-05-08T06:38:40.825440+0000) 2024-05-08T06:38:51.980 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:51 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:51.712+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2024-05-08T06:38:21.926513+0000 front 2024-05-08T06:38:21.926485+0000 (oldest deadline 2024-05-08T06:38:48.927152+0000) 2024-05-08T06:38:51.980 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:51 smithi134 systemd[1]: Stopping Ceph osd.7 for 02181fb0-0d04-11ef-bc97-c7b262605968... 2024-05-08T06:38:51.980 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:51 smithi134 bash[41843]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd.7" found: no such container 2024-05-08T06:38:51.980 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:51 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:51.980+0000 7f4659247700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-08T06:38:51.980 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:51 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:51.980+0000 7f4659247700 -1 osd.7 70 *** Got signal Terminated *** 2024-05-08T06:38:51.981 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:51 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:51.980+0000 7f4659247700 -1 osd.7 70 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-08T06:38:52.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 3.23 MB/sec execute 73 sec latency 56459.286 ms 2024-05-08T06:38:53.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:52 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:52.760+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:53.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:52 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:52.760+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:53.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:52 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:52.760+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.131386+0000 front 2024-05-08T06:38:16.125303+0000 (oldest deadline 2024-05-08T06:38:40.825440+0000) 2024-05-08T06:38:53.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:52 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:52.760+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2024-05-08T06:38:21.926513+0000 front 2024-05-08T06:38:21.926485+0000 (oldest deadline 2024-05-08T06:38:48.927152+0000) 2024-05-08T06:38:53.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 3.19 MB/sec execute 74 sec latency 57459.333 ms 2024-05-08T06:38:54.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:53 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:53.722+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:54.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:53 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:53.722+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:54.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:53 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:53.722+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.131386+0000 front 2024-05-08T06:38:16.125303+0000 (oldest deadline 2024-05-08T06:38:40.825440+0000) 2024-05-08T06:38:54.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:53 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:53.722+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2024-05-08T06:38:21.926513+0000 front 2024-05-08T06:38:21.926485+0000 (oldest deadline 2024-05-08T06:38:48.927152+0000) 2024-05-08T06:38:54.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 3.15 MB/sec execute 75 sec latency 58459.387 ms 2024-05-08T06:38:55.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:54 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:54.751+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:55.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:54 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:54.751+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:55.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:54 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:54.751+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.131386+0000 front 2024-05-08T06:38:16.125303+0000 (oldest deadline 2024-05-08T06:38:40.825440+0000) 2024-05-08T06:38:55.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:54 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:54.751+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2024-05-08T06:38:21.926513+0000 front 2024-05-08T06:38:21.926485+0000 (oldest deadline 2024-05-08T06:38:48.927152+0000) 2024-05-08T06:38:55.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 3.11 MB/sec execute 76 sec latency 59459.427 ms 2024-05-08T06:38:56.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:55 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:55.734+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:56.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:55 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:55.734+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:56.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:55 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:55.734+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.131386+0000 front 2024-05-08T06:38:16.125303+0000 (oldest deadline 2024-05-08T06:38:40.825440+0000) 2024-05-08T06:38:56.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:55 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:55.734+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2024-05-08T06:38:21.926513+0000 front 2024-05-08T06:38:21.926485+0000 (oldest deadline 2024-05-08T06:38:48.927152+0000) 2024-05-08T06:38:56.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:55 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:55.734+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.134:6804 osd.4 since back 2024-05-08T06:38:28.927694+0000 front 2024-05-08T06:38:28.927542+0000 (oldest deadline 2024-05-08T06:38:55.328068+0000) 2024-05-08T06:38:56.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 3.07 MB/sec execute 77 sec latency 60459.482 ms 2024-05-08T06:38:57.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:56 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:56.697+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6806 osd.0 since back 2024-05-08T06:38:01.423517+0000 front 2024-05-08T06:38:01.431324+0000 (oldest deadline 2024-05-08T06:38:24.923718+0000) 2024-05-08T06:38:57.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:56 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:56.697+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6814 osd.1 since back 2024-05-08T06:38:04.924177+0000 front 2024-05-08T06:38:04.929033+0000 (oldest deadline 2024-05-08T06:38:30.824329+0000) 2024-05-08T06:38:57.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:56 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:56.697+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6822 osd.2 since back 2024-05-08T06:38:16.131386+0000 front 2024-05-08T06:38:16.125303+0000 (oldest deadline 2024-05-08T06:38:40.825440+0000) 2024-05-08T06:38:57.028 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:56 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:56.697+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.82:6830 osd.3 since back 2024-05-08T06:38:21.926513+0000 front 2024-05-08T06:38:21.926485+0000 (oldest deadline 2024-05-08T06:38:48.927152+0000) 2024-05-08T06:38:57.029 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:56 smithi134 ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7[38271]: 2024-05-08T06:38:56.697+0000 7f46523db700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.134:6804 osd.4 since back 2024-05-08T06:38:28.927694+0000 front 2024-05-08T06:38:28.927542+0000 (oldest deadline 2024-05-08T06:38:55.328068+0000) 2024-05-08T06:38:57.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 3.03 MB/sec execute 78 sec latency 61459.506 ms 2024-05-08T06:38:57.895 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:57 smithi134 podman[41854]: 2024-05-08 06:38:57.553527491 +0000 UTC m=+5.676035463 container died 9b1f288b1e8cf7b21529a0a89cd1615f567de2264a1e7206e8fafcb617c4f230 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, release=754, vcs-type=git, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, ceph=True, io.k8s.display-name=CentOS Stream 8, architecture=x86_64) 2024-05-08T06:38:57.895 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:57 smithi134 podman[41854]: 2024-05-08 06:38:57.896188115 +0000 UTC m=+6.018696094 container cleanup 9b1f288b1e8cf7b21529a0a89cd1615f567de2264a1e7206e8fafcb617c4f230 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7, release=754, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, RELEASE=HEAD, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-05-08T06:38:58.278 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:57 smithi134 bash[41854]: ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7 2024-05-08T06:38:58.278 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:58 smithi134 podman[41873]: 2024-05-08 06:38:58.080869365 +0000 UTC m=+0.525796476 container remove 9b1f288b1e8cf7b21529a0a89cd1615f567de2264a1e7206e8fafcb617c4f230 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, name=centos-stream, GIT_BRANCH=HEAD, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , distribution-scope=public, com.redhat.component=centos-stream-container, ceph=True, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-05-08T06:38:58.278 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:58 smithi134 bash[41886]: Error: no container with name or ID "ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd.7" found: no such container 2024-05-08T06:38:58.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.99 MB/sec execute 79 sec latency 62459.553 ms 2024-05-08T06:38:58.640 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:58 smithi134 podman[41947]: 2024-05-08 06:38:58.24810533 +0000 UTC m=+0.018816682 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-08T06:38:58.641 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:58 smithi134 podman[41947]: 2024-05-08 06:38:58.356550551 +0000 UTC m=+0.127261888 container create bbe613955fd836f1853008216df7490fa3f9f54beeb4031f9184d5b319ef6e21 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7-deactivate, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, release=754, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, name=centos-stream) 2024-05-08T06:38:58.641 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:58 smithi134 podman[41947]: 2024-05-08 06:38:58.642355436 +0000 UTC m=+0.413066783 container init bbe613955fd836f1853008216df7490fa3f9f54beeb4031f9184d5b319ef6e21 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7-deactivate, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , architecture=x86_64, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, ceph=True, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, name=centos-stream, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-05-08T06:38:58.923 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:58 smithi134 podman[41947]: 2024-05-08 06:38:58.646109708 +0000 UTC m=+0.416821050 container start bbe613955fd836f1853008216df7490fa3f9f54beeb4031f9184d5b319ef6e21 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7-deactivate, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, ceph=True, version=8, GIT_CLEAN=True, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, distribution-scope=public, com.redhat.component=centos-stream-container) 2024-05-08T06:38:58.924 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:58 smithi134 podman[41947]: 2024-05-08 06:38:58.725734608 +0000 UTC m=+0.496445957 container attach bbe613955fd836f1853008216df7490fa3f9f54beeb4031f9184d5b319ef6e21 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7-deactivate, ceph=True, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, name=centos-stream, distribution-scope=public, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git) 2024-05-08T06:38:59.218 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:58 smithi134 podman[41947]: 2024-05-08 06:38:58.925565371 +0000 UTC m=+0.696276715 container died bbe613955fd836f1853008216df7490fa3f9f54beeb4031f9184d5b319ef6e21 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7-deactivate, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, ceph=True, RELEASE=HEAD, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, name=centos-stream, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc.) 2024-05-08T06:38:59.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.95 MB/sec execute 80 sec latency 63459.609 ms 2024-05-08T06:38:59.470 INFO:journalctl@ceph.osd.7.smithi134.stdout:May 08 06:38:59 smithi134 podman[41973]: 2024-05-08 06:38:59.461921578 +0000 UTC m=+0.530883652 container remove bbe613955fd836f1853008216df7490fa3f9f54beeb4031f9184d5b319ef6e21 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-02181fb0-0d04-11ef-bc97-c7b262605968-osd-7-deactivate, architecture=x86_64, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, release=754, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, RELEASE=HEAD, distribution-scope=public) 2024-05-08T06:38:59.476 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-02181fb0-0d04-11ef-bc97-c7b262605968@osd.7.service' 2024-05-08T06:38:59.514 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-08T06:38:59.515 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-05-08T06:38:59.515 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 --force --keep-logs 2024-05-08T06:39:00.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.92 MB/sec execute 81 sec latency 64459.662 ms 2024-05-08T06:39:01.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.88 MB/sec execute 82 sec latency 65459.688 ms 2024-05-08T06:39:02.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.84 MB/sec execute 83 sec latency 66459.712 ms 2024-05-08T06:39:03.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.81 MB/sec execute 84 sec latency 67459.735 ms 2024-05-08T06:39:04.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.78 MB/sec execute 85 sec latency 68459.758 ms 2024-05-08T06:39:05.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.75 MB/sec execute 86 sec latency 69459.787 ms 2024-05-08T06:39:06.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.71 MB/sec execute 87 sec latency 70459.842 ms 2024-05-08T06:39:07.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.68 MB/sec execute 88 sec latency 71459.890 ms 2024-05-08T06:39:08.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.65 MB/sec execute 89 sec latency 72460.092 ms 2024-05-08T06:39:09.435 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.62 MB/sec execute 90 sec latency 73460.127 ms 2024-05-08T06:39:10.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.59 MB/sec execute 91 sec latency 74460.151 ms 2024-05-08T06:39:11.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.57 MB/sec execute 92 sec latency 75460.216 ms 2024-05-08T06:39:12.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.54 MB/sec execute 93 sec latency 76460.238 ms 2024-05-08T06:39:13.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.51 MB/sec execute 94 sec latency 77460.288 ms 2024-05-08T06:39:14.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.49 MB/sec execute 95 sec latency 78460.345 ms 2024-05-08T06:39:15.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.46 MB/sec execute 96 sec latency 79460.364 ms 2024-05-08T06:39:16.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.43 MB/sec execute 97 sec latency 80460.424 ms 2024-05-08T06:39:17.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.41 MB/sec execute 98 sec latency 81460.467 ms 2024-05-08T06:39:18.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.39 MB/sec execute 99 sec latency 82460.483 ms 2024-05-08T06:39:19.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.36 MB/sec execute 100 sec latency 83460.496 ms 2024-05-08T06:39:20.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.34 MB/sec execute 101 sec latency 84460.535 ms 2024-05-08T06:39:21.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.32 MB/sec execute 102 sec latency 85460.585 ms 2024-05-08T06:39:22.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.29 MB/sec execute 103 sec latency 86460.623 ms 2024-05-08T06:39:23.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.27 MB/sec execute 104 sec latency 87460.664 ms 2024-05-08T06:39:24.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.25 MB/sec execute 105 sec latency 88460.695 ms 2024-05-08T06:39:25.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.23 MB/sec execute 106 sec latency 89460.733 ms 2024-05-08T06:39:26.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.21 MB/sec execute 107 sec latency 90460.751 ms 2024-05-08T06:39:27.351 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 --force --keep-logs 2024-05-08T06:39:27.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.19 MB/sec execute 108 sec latency 91460.805 ms 2024-05-08T06:39:28.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.17 MB/sec execute 109 sec latency 92460.836 ms 2024-05-08T06:39:29.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.15 MB/sec execute 110 sec latency 93460.858 ms 2024-05-08T06:39:30.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.13 MB/sec execute 111 sec latency 94460.899 ms 2024-05-08T06:39:31.436 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.11 MB/sec execute 112 sec latency 95460.946 ms 2024-05-08T06:39:32.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.09 MB/sec execute 113 sec latency 96461.043 ms 2024-05-08T06:39:33.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.07 MB/sec execute 114 sec latency 97461.107 ms 2024-05-08T06:39:34.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.05 MB/sec execute 115 sec latency 98461.164 ms 2024-05-08T06:39:35.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.04 MB/sec execute 116 sec latency 99461.266 ms 2024-05-08T06:39:36.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.02 MB/sec execute 117 sec latency 100461.336 ms 2024-05-08T06:39:37.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 2.00 MB/sec execute 118 sec latency 101461.364 ms 2024-05-08T06:39:38.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.98 MB/sec execute 119 sec latency 102461.399 ms 2024-05-08T06:39:39.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.97 MB/sec execute 120 sec latency 103461.433 ms 2024-05-08T06:39:40.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.95 MB/sec execute 121 sec latency 104461.456 ms 2024-05-08T06:39:41.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.94 MB/sec execute 122 sec latency 105461.519 ms 2024-05-08T06:39:42.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.92 MB/sec execute 123 sec latency 106461.558 ms 2024-05-08T06:39:43.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.90 MB/sec execute 124 sec latency 107461.620 ms 2024-05-08T06:39:44.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.89 MB/sec execute 125 sec latency 108461.685 ms 2024-05-08T06:39:45.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.87 MB/sec execute 126 sec latency 109461.720 ms 2024-05-08T06:39:46.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.86 MB/sec execute 127 sec latency 110461.756 ms 2024-05-08T06:39:46.555 DEBUG:teuthology.orchestra.run.smithi082:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-08T06:39:46.588 DEBUG:teuthology.orchestra.run.smithi134:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-08T06:39:46.616 INFO:tasks.cephadm:Archiving crash dumps... 2024-05-08T06:39:46.618 DEBUG:teuthology.misc:Transferring archived files from smithi082:/var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/crash to /home/teuthworker/archive/adking-2024-05-08_05:24:58-orch:cephadm-wip-adk-testing-2024-05-07-0802-distro-default-smithi/7697317/remote/smithi082/crash 2024-05-08T06:39:46.619 DEBUG:teuthology.orchestra.run.smithi082:> sudo tar c -f - -C /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/crash -- . 2024-05-08T06:39:46.665 INFO:teuthology.orchestra.run.smithi082.stderr:tar: /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/crash: Cannot open: No such file or directory 2024-05-08T06:39:46.666 INFO:teuthology.orchestra.run.smithi082.stderr:tar: Error is not recoverable: exiting now 2024-05-08T06:39:46.668 DEBUG:teuthology.misc:Transferring archived files from smithi134:/var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/crash to /home/teuthworker/archive/adking-2024-05-08_05:24:58-orch:cephadm-wip-adk-testing-2024-05-07-0802-distro-default-smithi/7697317/remote/smithi134/crash 2024-05-08T06:39:46.669 DEBUG:teuthology.orchestra.run.smithi134:> sudo tar c -f - -C /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/crash -- . 2024-05-08T06:39:46.735 INFO:teuthology.orchestra.run.smithi134.stderr:tar: /var/lib/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/crash: Cannot open: No such file or directory 2024-05-08T06:39:46.736 INFO:teuthology.orchestra.run.smithi134.stderr:tar: Error is not recoverable: exiting now 2024-05-08T06:39:46.736 INFO:tasks.cephadm:Checking cluster log for badness... 2024-05-08T06:39:46.737 DEBUG:teuthology.orchestra.run.smithi082:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-05-08T06:39:46.766 INFO:tasks.cephadm:Compressing logs... 2024-05-08T06:39:46.766 DEBUG:teuthology.orchestra.run.smithi082:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-05-08T06:39:46.808 DEBUG:teuthology.orchestra.run.smithi134:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-05-08T06:39:46.833 INFO:teuthology.orchestra.run.smithi082.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-mgr.smithi082.bsnvtb.log 2024-05-08T06:39:46.833 INFO:teuthology.orchestra.run.smithi082.stderr:: No such file or directory 2024-05-08T06:39:46.833 INFO:teuthology.orchestra.run.smithi082.stderr:gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph.log 2024-05-08T06:39:46.833 INFO:teuthology.orchestra.run.smithi082.stderr:/var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-mgr.smithi082.bsnvtb.log: gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.2.log 2024-05-08T06:39:46.833 INFO:teuthology.orchestra.run.smithi082.stderr:/var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-mds.foofs.smithi082.ofnfgy.log 2024-05-08T06:39:46.833 INFO:teuthology.orchestra.run.smithi082.stderr:/var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph.audit.log 2024-05-08T06:39:46.840 INFO:teuthology.orchestra.run.smithi082.stderr:gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.3.log 2024-05-08T06:39:46.840 INFO:teuthology.orchestra.run.smithi082.stderr:/var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-mds.foofs.smithi082.ofnfgy.log: /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph.audit.log: 80.3% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-mds.foofs.smithi082.ofnfgy.log.gz 2024-05-08T06:39:46.840 INFO:teuthology.orchestra.run.smithi082.stderr:gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.1.log 2024-05-08T06:39:46.840 INFO:teuthology.orchestra.run.smithi082.stderr: 86.8% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph.log.gz 2024-05-08T06:39:46.840 INFO:teuthology.orchestra.run.smithi082.stderr:/var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.0.log 2024-05-08T06:39:46.840 INFO:teuthology.orchestra.run.smithi082.stderr:gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph.cephadm.log 2024-05-08T06:39:46.841 INFO:teuthology.orchestra.run.smithi082.stderr:/var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.1.log: /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-volume.log 2024-05-08T06:39:46.841 INFO:teuthology.orchestra.run.smithi082.stderr:/var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-mon.smithi082.log 2024-05-08T06:39:46.841 INFO:teuthology.orchestra.run.smithi082.stderr: 83.0% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph.cephadm.log.gz 2024-05-08T06:39:46.841 INFO:teuthology.orchestra.run.smithi082.stderr:/var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-volume.log: 90.8% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph.audit.log.gzgzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-05-08T06:39:46.841 INFO:teuthology.orchestra.run.smithi082.stderr: 2024-05-08T06:39:46.843 INFO:teuthology.orchestra.run.smithi082.stderr:/var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-mon.smithi082.log: /var/log/ceph/cephadm.log: 90.8% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-volume.log.gz 2024-05-08T06:39:46.844 INFO:teuthology.orchestra.run.smithi134.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph.log 2024-05-08T06:39:46.844 INFO:teuthology.orchestra.run.smithi134.stderr:: No such file or directory 2024-05-08T06:39:46.844 INFO:teuthology.orchestra.run.smithi134.stderr:gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph.audit.log 2024-05-08T06:39:46.844 INFO:teuthology.orchestra.run.smithi134.stderr:gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.7.log 2024-05-08T06:39:46.844 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph.log: /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.5.log 2024-05-08T06:39:46.844 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.4.log 2024-05-08T06:39:46.844 INFO:teuthology.orchestra.run.smithi134.stderr:gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-mon.smithi134.log 2024-05-08T06:39:46.845 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.5.log: 86.8% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph.log.gz 2024-05-08T06:39:46.845 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-mgr.smithi134.zbsjta.log 2024-05-08T06:39:46.845 INFO:teuthology.orchestra.run.smithi134.stderr: 91.0% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph.audit.log.gz 2024-05-08T06:39:46.845 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-mon.smithi134.log: gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.6.log 2024-05-08T06:39:46.845 INFO:teuthology.orchestra.run.smithi134.stderr:gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph.cephadm.log 2024-05-08T06:39:46.845 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-mgr.smithi134.zbsjta.log: /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-mds.foofs.smithi134.ooovgp.log 2024-05-08T06:39:46.845 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-volume.log 2024-05-08T06:39:46.845 INFO:teuthology.orchestra.run.smithi134.stderr: 81.9% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph.cephadm.log.gz 2024-05-08T06:39:46.845 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-mds.foofs.smithi134.ooovgp.log: 60.5% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-mds.foofs.smithi134.ooovgp.log.gz 2024-05-08T06:39:46.846 INFO:teuthology.orchestra.run.smithi134.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-05-08T06:39:46.846 INFO:teuthology.orchestra.run.smithi134.stderr:/var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-volume.log: /var/log/ceph/cephadm.log: 92.3% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-mgr.smithi134.zbsjta.log.gz 2024-05-08T06:39:46.846 INFO:teuthology.orchestra.run.smithi134.stderr: 90.7% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-volume.log.gz 2024-05-08T06:39:46.846 INFO:teuthology.orchestra.run.smithi134.stderr: 90.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-05-08T06:39:46.851 INFO:teuthology.orchestra.run.smithi082.stderr: 92.0% -- replaced with /var/log/ceph/cephadm.log.gz 2024-05-08T06:39:46.943 INFO:teuthology.orchestra.run.smithi082.stderr: 88.9% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-mgr.smithi082.bsnvtb.log.gz 2024-05-08T06:39:46.974 INFO:teuthology.orchestra.run.smithi134.stderr: 92.6% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-mon.smithi134.log.gz 2024-05-08T06:39:47.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.84 MB/sec execute 128 sec latency 111461.801 ms 2024-05-08T06:39:47.478 INFO:teuthology.orchestra.run.smithi082.stderr: 92.0% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-mon.smithi082.log.gz 2024-05-08T06:39:48.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.83 MB/sec execute 129 sec latency 112461.841 ms 2024-05-08T06:39:49.320 INFO:teuthology.orchestra.run.smithi082.stderr: 94.6% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.2.log.gz 2024-05-08T06:39:49.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.82 MB/sec execute 130 sec latency 113461.903 ms 2024-05-08T06:39:50.438 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.80 MB/sec execute 131 sec latency 114461.942 ms 2024-05-08T06:39:50.932 INFO:teuthology.orchestra.run.smithi134.stderr: 94.3% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.6.log.gz 2024-05-08T06:39:51.036 INFO:teuthology.orchestra.run.smithi134.stderr: 94.0% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.4.log.gz 2024-05-08T06:39:51.372 INFO:teuthology.orchestra.run.smithi082.stderr: 94.1% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.3.log.gz 2024-05-08T06:39:51.428 INFO:teuthology.orchestra.run.smithi082.stderr: 94.3% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.1.log.gz 2024-05-08T06:39:51.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.79 MB/sec execute 132 sec latency 115461.984 ms 2024-05-08T06:39:51.583 INFO:teuthology.orchestra.run.smithi082.stderr: 94.2% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.0.log.gz 2024-05-08T06:39:51.585 INFO:teuthology.orchestra.run.smithi082.stderr: 2024-05-08T06:39:51.585 INFO:teuthology.orchestra.run.smithi082.stderr:real 0m4.764s 2024-05-08T06:39:51.586 INFO:teuthology.orchestra.run.smithi082.stderr:user 0m16.475s 2024-05-08T06:39:51.586 INFO:teuthology.orchestra.run.smithi082.stderr:sys 0m0.669s 2024-05-08T06:39:52.147 INFO:teuthology.orchestra.run.smithi134.stderr: 94.3% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.5.log.gz 2024-05-08T06:39:52.437 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.78 MB/sec execute 133 sec latency 116462.052 ms 2024-05-08T06:39:53.438 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.76 MB/sec execute 134 sec latency 117462.128 ms 2024-05-08T06:39:54.156 INFO:teuthology.orchestra.run.smithi134.stderr: 94.2% -- replaced with /var/log/ceph/02181fb0-0d04-11ef-bc97-c7b262605968/ceph-osd.7.log.gz 2024-05-08T06:39:54.158 INFO:teuthology.orchestra.run.smithi134.stderr: 2024-05-08T06:39:54.158 INFO:teuthology.orchestra.run.smithi134.stderr:real 0m7.334s 2024-05-08T06:39:54.158 INFO:teuthology.orchestra.run.smithi134.stderr:user 0m20.274s 2024-05-08T06:39:54.158 INFO:teuthology.orchestra.run.smithi134.stderr:sys 0m0.807s 2024-05-08T06:39:54.158 INFO:tasks.cephadm:Archiving logs... 2024-05-08T06:39:54.159 DEBUG:teuthology.misc:Transferring archived files from smithi082:/var/log/ceph to /home/teuthworker/archive/adking-2024-05-08_05:24:58-orch:cephadm-wip-adk-testing-2024-05-07-0802-distro-default-smithi/7697317/remote/smithi082/log 2024-05-08T06:39:54.160 DEBUG:teuthology.orchestra.run.smithi082:> sudo tar c -f - -C /var/log/ceph -- . 2024-05-08T06:39:54.458 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.75 MB/sec execute 135 sec latency 118462.177 ms 2024-05-08T06:39:55.453 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.74 MB/sec execute 136 sec latency 119462.246 ms 2024-05-08T06:39:56.458 DEBUG:teuthology.misc:Transferring archived files from smithi134:/var/log/ceph to /home/teuthworker/archive/adking-2024-05-08_05:24:58-orch:cephadm-wip-adk-testing-2024-05-07-0802-distro-default-smithi/7697317/remote/smithi134/log 2024-05-08T06:39:56.459 DEBUG:teuthology.orchestra.run.smithi134:> sudo tar c -f - -C /var/log/ceph -- . 2024-05-08T06:39:56.460 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.72 MB/sec execute 137 sec latency 120462.289 ms 2024-05-08T06:39:57.454 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.71 MB/sec execute 138 sec latency 121462.353 ms 2024-05-08T06:39:58.452 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.70 MB/sec execute 139 sec latency 122462.453 ms 2024-05-08T06:39:59.411 INFO:tasks.cephadm:Removing cluster... 2024-05-08T06:39:59.411 DEBUG:teuthology.orchestra.run.smithi082:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 --force 2024-05-08T06:39:59.438 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.69 MB/sec execute 140 sec latency 123462.493 ms 2024-05-08T06:39:59.712 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 --force 2024-05-08T06:40:00.034 INFO:tasks.cephadm:Removing cephadm ... 2024-05-08T06:40:00.035 DEBUG:teuthology.orchestra.run.smithi082:> rm -rf /home/ubuntu/cephtest/cephadm 2024-05-08T06:40:00.051 DEBUG:teuthology.orchestra.run.smithi134:> rm -rf /home/ubuntu/cephtest/cephadm 2024-05-08T06:40:00.065 INFO:tasks.cephadm:Teardown complete 2024-05-08T06:40:00.066 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-05-08T06:40:00.076 INFO:teuthology.task.clock:Checking final clock skew... 2024-05-08T06:40:00.076 DEBUG:teuthology.orchestra.run.smithi082:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-08T06:40:00.093 DEBUG:teuthology.orchestra.run.smithi134:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-08T06:40:00.105 INFO:teuthology.orchestra.run.smithi082.stderr:bash: line 1: ntpq: command not found 2024-05-08T06:40:00.109 INFO:teuthology.orchestra.run.smithi082.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-08T06:40:00.109 INFO:teuthology.orchestra.run.smithi082.stdout:=============================================================================== 2024-05-08T06:40:00.109 INFO:teuthology.orchestra.run.smithi082.stdout:^* hv01.front.sepia.ceph.com 3 6 377 179 +542us[ +598us] +/- 54ms 2024-05-08T06:40:00.109 INFO:teuthology.orchestra.run.smithi082.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 48 -1674us[-1674us] +/- 66ms 2024-05-08T06:40:00.109 INFO:teuthology.orchestra.run.smithi082.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 50 +953us[ +953us] +/- 59ms 2024-05-08T06:40:00.109 INFO:teuthology.orchestra.run.smithi082.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-05-08T06:40:00.120 INFO:teuthology.orchestra.run.smithi134.stderr:bash: line 1: ntpq: command not found 2024-05-08T06:40:00.124 INFO:teuthology.orchestra.run.smithi134.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-08T06:40:00.124 INFO:teuthology.orchestra.run.smithi134.stdout:=============================================================================== 2024-05-08T06:40:00.124 INFO:teuthology.orchestra.run.smithi134.stdout:^* hv01.front.sepia.ceph.com 3 6 377 46 +505us[ +501us] +/- 56ms 2024-05-08T06:40:00.124 INFO:teuthology.orchestra.run.smithi134.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 50 -1633us[-1637us] +/- 66ms 2024-05-08T06:40:00.124 INFO:teuthology.orchestra.run.smithi134.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 47 +1002us[ +998us] +/- 60ms 2024-05-08T06:40:00.125 INFO:teuthology.orchestra.run.smithi134.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-05-08T06:40:00.126 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-05-08T06:40:00.136 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-05-08T06:40:00.138 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-05-08T06:40:00.148 DEBUG:teuthology.orchestra.run.smithi082:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-05-08T06:40:00.222 DEBUG:teuthology.orchestra.run.smithi134:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-05-08T06:40:00.287 DEBUG:teuthology.orchestra.run.smithi082:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-05-08T06:40:00.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-08T06:40:00.316 DEBUG:teuthology.orchestra.run.smithi134:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-05-08T06:40:00.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-08T06:40:00.345 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-05-08T06:40:00.355 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-05-08T06:40:00.369 INFO:teuthology.task.internal:Duration was 1054.306235 seconds 2024-05-08T06:40:00.370 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-05-08T06:40:00.379 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-05-08T06:40:00.380 DEBUG:teuthology.orchestra.run.smithi082:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-05-08T06:40:00.382 DEBUG:teuthology.orchestra.run.smithi134:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-05-08T06:40:00.421 INFO:teuthology.orchestra.run.smithi082.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-08T06:40:00.423 INFO:teuthology.orchestra.run.smithi134.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-08T06:40:00.438 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.67 MB/sec execute 141 sec latency 124462.523 ms 2024-05-08T06:40:00.718 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-05-08T06:40:00.718 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi082.front.sepia.ceph.com 2024-05-08T06:40:00.718 DEBUG:teuthology.orchestra.run.smithi082:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-05-08T06:40:00.738 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi134.front.sepia.ceph.com 2024-05-08T06:40:00.738 DEBUG:teuthology.orchestra.run.smithi134:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-05-08T06:40:00.778 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-05-08T06:40:00.778 DEBUG:teuthology.orchestra.run.smithi082:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-05-08T06:40:00.781 DEBUG:teuthology.orchestra.run.smithi134:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-05-08T06:40:00.846 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-05-08T06:40:00.846 DEBUG:teuthology.orchestra.run.smithi082:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-05-08T06:40:00.848 DEBUG:teuthology.orchestra.run.smithi134:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-05-08T06:40:01.438 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.66 MB/sec execute 142 sec latency 125462.557 ms 2024-05-08T06:40:02.438 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.65 MB/sec execute 143 sec latency 126462.624 ms 2024-05-08T06:40:03.438 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.64 MB/sec execute 144 sec latency 127462.655 ms 2024-05-08T06:40:04.438 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.63 MB/sec execute 145 sec latency 128462.688 ms 2024-05-08T06:40:05.438 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.62 MB/sec execute 146 sec latency 129462.717 ms 2024-05-08T06:40:05.836 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-05-08T06:40:05.846 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-05-08T06:40:05.847 DEBUG:teuthology.orchestra.run.smithi082:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-05-08T06:40:05.872 DEBUG:teuthology.orchestra.run.smithi134:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-05-08T06:40:05.902 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-05-08T06:40:05.913 DEBUG:teuthology.orchestra.run.smithi082:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-05-08T06:40:05.916 DEBUG:teuthology.orchestra.run.smithi134:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-05-08T06:40:05.936 INFO:teuthology.orchestra.run.smithi082.stdout:kernel.core_pattern = core 2024-05-08T06:40:05.970 INFO:teuthology.orchestra.run.smithi134.stdout:kernel.core_pattern = core 2024-05-08T06:40:06.002 DEBUG:teuthology.orchestra.run.smithi082:> test -e /home/ubuntu/cephtest/archive/coredump 2024-05-08T06:40:06.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-08T06:40:06.033 DEBUG:teuthology.orchestra.run.smithi134:> test -e /home/ubuntu/cephtest/archive/coredump 2024-05-08T06:40:06.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-08T06:40:06.058 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-05-08T06:40:06.068 INFO:teuthology.task.internal:Transferring archived files... 2024-05-08T06:40:06.069 DEBUG:teuthology.misc:Transferring archived files from smithi082:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-05-08_05:24:58-orch:cephadm-wip-adk-testing-2024-05-07-0802-distro-default-smithi/7697317/remote/smithi082 2024-05-08T06:40:06.069 DEBUG:teuthology.orchestra.run.smithi082:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-05-08T06:40:06.122 DEBUG:teuthology.misc:Transferring archived files from smithi134:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-05-08_05:24:58-orch:cephadm-wip-adk-testing-2024-05-07-0802-distro-default-smithi/7697317/remote/smithi134 2024-05-08T06:40:06.122 DEBUG:teuthology.orchestra.run.smithi134:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-05-08T06:40:06.162 INFO:teuthology.task.internal:Removing archive directory... 2024-05-08T06:40:06.162 DEBUG:teuthology.orchestra.run.smithi082:> rm -rf -- /home/ubuntu/cephtest/archive 2024-05-08T06:40:06.165 DEBUG:teuthology.orchestra.run.smithi134:> rm -rf -- /home/ubuntu/cephtest/archive 2024-05-08T06:40:06.212 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-05-08T06:40:06.223 INFO:teuthology.task.internal:Not uploading archives. 2024-05-08T06:40:06.224 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-05-08T06:40:06.247 INFO:teuthology.task.internal:Tidying up after the test... 2024-05-08T06:40:06.247 DEBUG:teuthology.orchestra.run.smithi082:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-05-08T06:40:06.249 DEBUG:teuthology.orchestra.run.smithi134:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-05-08T06:40:06.262 INFO:teuthology.orchestra.run.smithi082.stdout: 262165 4 drwxr-xr-x 2 ubuntu ubuntu 4096 May 8 06:40 /home/ubuntu/cephtest 2024-05-08T06:40:06.268 INFO:teuthology.orchestra.run.smithi134.stdout: 265585 4 drwxr-xr-x 2 ubuntu ubuntu 4096 May 8 06:40 /home/ubuntu/cephtest 2024-05-08T06:40:06.269 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-05-08T06:40:06.280 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-05-08T06:40:06.354 INFO:teuthology.nuke:Checking targets against current locks 2024-05-08T06:40:06.382 DEBUG:teuthology.nuke:shortname: smithi082 2024-05-08T06:40:06.383 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-05-08T06:40:06.408 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi082.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-05-08_05:24:58-orch:cephadm-wip-adk-testing-2024-05-07-0802-distro-default-smithi/7697317', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-08 06:13:39.939450', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGY1qVRgCmeZi86MEu9yJJhTrNkjNOU1rLfL+l8aLBNLXBXdIFVxBJkgsRCDREIVpXQU1QE2qkNu7HyIzfj7E1M='} 2024-05-08T06:40:06.412 DEBUG:teuthology.nuke:shortname: smithi134 2024-05-08T06:40:06.413 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-05-08T06:40:06.439 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi134.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-05-08_05:24:58-orch:cephadm-wip-adk-testing-2024-05-07-0802-distro-default-smithi/7697317', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-08 06:13:39.940982', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBChUJ8uXPx39LBsFglaZqzd30QXjfWAQm3+k2zF+FcKuUVlBC6Shopwj017QS7+eyg8ge8/tOnMOdKN+pOEjXBs='} 2024-05-08T06:40:06.442 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.61 MB/sec execute 147 sec latency 130462.782 ms 2024-05-08T06:40:06.473 INFO:teuthology.orchestra.console.smithi082:Power off 2024-05-08T06:40:06.514 DEBUG:teuthology.orchestra.console.smithi082:pexpect command: ipmitool -H smithi082.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-05-08T06:40:06.544 DEBUG:teuthology.orchestra.console.smithi082:power off output: Chassis Power Control: Down/Off 2024-05-08T06:40:06.544 DEBUG:teuthology.orchestra.console.smithi082:pexpect command: ipmitool -H smithi082.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-08T06:40:06.568 INFO:teuthology.orchestra.console.smithi134:Power off 2024-05-08T06:40:06.568 DEBUG:teuthology.orchestra.console.smithi134:pexpect command: ipmitool -H smithi134.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-05-08T06:40:06.585 DEBUG:teuthology.orchestra.console.smithi082:check power output: Chassis Power is on 2024-05-08T06:40:06.600 DEBUG:teuthology.orchestra.console.smithi134:power off output: Chassis Power Control: Down/Off 2024-05-08T06:40:06.600 DEBUG:teuthology.orchestra.console.smithi134:pexpect command: ipmitool -H smithi134.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-08T06:40:06.669 DEBUG:teuthology.orchestra.console.smithi134:check power output: Chassis Power is on 2024-05-08T06:40:07.439 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.60 MB/sec execute 148 sec latency 131462.861 ms 2024-05-08T06:40:08.439 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.58 MB/sec execute 149 sec latency 132462.959 ms 2024-05-08T06:40:09.439 INFO:teuthology.orchestra.run.smithi082.stdout: 5 84388 1.57 MB/sec execute 150 sec latency 133463.048 ms 2024-05-08T06:40:09.639 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-08T06:40:10.586 DEBUG:teuthology.orchestra.console.smithi082:pexpect command: ipmitool -H smithi082.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-08T06:40:10.671 DEBUG:teuthology.orchestra.console.smithi134:pexpect command: ipmitool -H smithi134.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-08T06:40:10.697 DEBUG:teuthology.orchestra.console.smithi082:check power output: Chassis Power is on 2024-05-08T06:40:10.783 DEBUG:teuthology.orchestra.console.smithi134:check power output: Chassis Power is on 2024-05-08T06:40:14.702 DEBUG:teuthology.orchestra.console.smithi082:pexpect command: ipmitool -H smithi082.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-08T06:40:14.784 DEBUG:teuthology.orchestra.console.smithi134:pexpect command: ipmitool -H smithi134.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-05-08T06:40:14.813 DEBUG:teuthology.orchestra.console.smithi082:check power output: Chassis Power is off 2024-05-08T06:40:14.895 DEBUG:teuthology.orchestra.console.smithi134:check power output: Chassis Power is off 2024-05-08T06:40:14.913 INFO:teuthology.orchestra.console.smithi082:Power off completed 2024-05-08T06:40:14.996 INFO:teuthology.orchestra.console.smithi134:Power off completed 2024-05-08T06:40:15.101 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1054.3062353134155 failure_reason: 'Command failed on smithi082 with status 1: ''sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 02181fb0-0d04-11ef-bc97-c7b262605968 -e sha1=7c8f650b36e258f639fa4a83becade57cbfd2009 -- bash -c \''ceph versions | jq -e \''"\''"\''.overall | keys\''"\''"\'' | grep $sha1\''''' owner: scheduled_adking@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=6c9bfa2f067a4777baad0ad056dd31e4 status: fail success: false 2024-05-08T06:40:15.102 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-05-08T06:40:15.196 INFO:teuthology.run:FAIL